diff --git a/.github/labeler.yml b/.github/labeler.yml
index fb51f1732fa17..5a87927411e04 100644
--- a/.github/labeler.yml
+++ b/.github/labeler.yml
@@ -1,4 +1,5 @@
 version: 1
+appendOnly: true
 
 # Match title
 labels:
@@ -30,6 +31,8 @@ labels:
   title: "^cherry-pick.*"
 - label: "cherry-pick"
   title: "^cherry pick.*"
+- label: "cherry-pick"
+  title: "^cherrypick.*"
 
 # Match body
 - label: "breaking-change"
@@ -38,4 +41,4 @@ labels:
 # Match File changes
 - label: "ci/run-e2e-single-node-tests"
   files:
-  - "src\\/meta\\/.*.rs"
\ No newline at end of file
+  - "src\\/meta\\/.*.rs"
diff --git a/.github/pr-title-checker-config.json b/.github/pr-title-checker-config.json
index 2da13e6e86f5c..27727f7b390c9 100644
--- a/.github/pr-title-checker-config.json
+++ b/.github/pr-title-checker-config.json
@@ -4,7 +4,7 @@
     "color": "B60205"
   },
   "CHECKS": {
-    "regexp": "^(cherry pick|cherry-pick)?(| |:|: )+(feat|fix|test|refactor|chore|style|doc|perf|build|ci|revert|deprecate)(\\(.*\\))?:.*",
+    "regexp": "^(cherrypick|cherry pick|cherry-pick)?(| |:|: )+(feat|fix|test|refactor|chore|style|doc|perf|build|ci|revert|deprecate)(\\(.*\\))?:.*",
     "ignoreLabels" : ["ignore-title"]
   },
   "MESSAGES": {
diff --git a/.github/workflows/typo.yml b/.github/workflows/typo.yml
index bf2163cdb0299..50f0ce89ac06d 100644
--- a/.github/workflows/typo.yml
+++ b/.github/workflows/typo.yml
@@ -10,4 +10,4 @@ jobs:
       uses: actions/checkout@v3
 
     - name: Check spelling of the entire repository
-      uses: crate-ci/typos@v1.20.4
+      uses: crate-ci/typos@v1.23.2
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index ab8ba3d9d7eb9..cb54c1606356e 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -7,6 +7,10 @@ repos:
     hooks:
     -   id: end-of-file-fixer
     -   id: trailing-whitespace
+-   repo: https://github.com/crate-ci/typos
+    rev: v1.23.1
+    hooks:
+      - id: typos
 -   repo: local
     hooks:
     -   id: rustfmt
@@ -14,10 +18,6 @@ repos:
         entry: rustfmt --edition 2021
         language: system
         types: [rust]
-    -   id: typos
-        name: typos
-        entry: typos -w
-        language: system
     -   id: cargo sort
         name: cargo sort
         entry: cargo sort -g -w
diff --git a/.typos.toml b/.typos.toml
index a7b5570bb766d..4d4bbfca1c082 100644
--- a/.typos.toml
+++ b/.typos.toml
@@ -14,7 +14,9 @@ mosquitto = "mosquitto" # This is a MQTT broker.
 abd = "abd"
 iy = "iy"
 Pn = "Pn"
+
 [default.extend-identifiers]
+TABLE_SCHEM = "TABLE_SCHEM"
 
 [files]
 extend-exclude = [
@@ -30,4 +32,8 @@ extend-exclude = [
     "**/Cargo.toml",
     "**/go.mod",
     "**/go.sum",
+    # https://github.com/risingwavelabs/risingwave/blob/0ce6228df6a4da183ae91146f2cdfff1ca9cc6a7/src/common/src/cast/mod.rs#L30
+    # We don't want to fix "fals" here, but may want in other places.
+    # Ideally, we should just ignore that line: https://github.com/crate-ci/typos/issues/316
+    "src/common/src/cast/mod.rs",
 ]
diff --git a/Cargo.lock b/Cargo.lock
index 1e510466fb67a..41cdbf376a908 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2,16 +2,6 @@
 # It is not intended for manual editing.
 version = 3
 
-[[package]]
-name = "Inflector"
-version = "0.11.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3"
-dependencies = [
- "lazy_static",
- "regex",
-]
-
 [[package]]
 name = "addr2line"
 version = "0.21.0"
@@ -68,31 +58,6 @@ dependencies = [
  "subtle",
 ]
 
-[[package]]
-name = "aes-kw"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "69fa2b352dcefb5f7f3a5fb840e02665d311d878955380515e4fd50095dd3d8c"
-dependencies = [
- "aes",
-]
-
-[[package]]
-name = "aes-siv"
-version = "0.7.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7e08d0cdb774acd1e4dac11478b1a0c0d203134b2aab0ba25eb430de9b18f8b9"
-dependencies = [
- "aead",
- "aes",
- "cipher",
- "cmac",
- "ctr",
- "dbl",
- "digest",
- "zeroize",
-]
-
 [[package]]
 name = "ahash"
 version = "0.7.8"
@@ -933,56 +898,6 @@ dependencies = [
  "rquickjs",
 ]
 
-[[package]]
-name = "arrow-udf-js-deno"
-version = "0.0.1"
-source = "git+https://github.com/risingwavelabs/arrow-udf.git?rev=fa36365#fa3636559de986aa592da6e8b3fbfac7bdd4bb78"
-dependencies = [
- "anyhow",
- "arrow-array 50.0.0",
- "arrow-buffer 50.0.0",
- "arrow-data 50.0.0",
- "arrow-schema 50.0.0",
- "arrow-udf-js-deno-runtime",
- "async-trait",
- "deno_core",
- "futures",
- "futures-util",
- "libc",
- "serde",
- "serde_json",
- "tokio",
- "v8",
-]
-
-[[package]]
-name = "arrow-udf-js-deno-runtime"
-version = "0.0.1"
-source = "git+https://github.com/risingwavelabs/arrow-udf.git?rev=fa36365#fa3636559de986aa592da6e8b3fbfac7bdd4bb78"
-dependencies = [
- "anyhow",
- "deno_ast",
- "deno_console",
- "deno_core",
- "deno_crypto",
- "deno_fetch",
- "deno_http",
- "deno_io",
- "deno_net",
- "deno_tls",
- "deno_url",
- "deno_web",
- "deno_webidl",
- "deno_websocket",
- "hyper 0.14.27",
- "libc",
- "serde",
- "serde_json",
- "signal-hook-registry",
- "tokio",
- "v8",
-]
-
 [[package]]
 name = "arrow-udf-python"
 version = "0.2.0"
@@ -1024,18 +939,6 @@ version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9b34d609dfbaf33d6889b2b7106d3ca345eacad44200913df5ba02bfd31d2ba9"
 
-[[package]]
-name = "ast_node"
-version = "0.9.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c3e3e06ec6ac7d893a0db7127d91063ad7d9da8988f8a1a256f03729e6eec026"
-dependencies = [
- "proc-macro2",
- "quote",
- "swc_macros_common",
- "syn 2.0.66",
-]
-
 [[package]]
 name = "async-attributes"
 version = "1.1.2"
@@ -1076,7 +979,6 @@ version = "0.4.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bc2d0cfb2a7388d34f590e76686704c494ed7aaceed62ee1ba35cbf363abc2a5"
 dependencies = [
- "brotli 3.5.0",
  "bzip2",
  "flate2",
  "futures-core",
@@ -1500,6 +1402,28 @@ dependencies = [
  "tracing",
 ]
 
+[[package]]
+name = "aws-sdk-glue"
+version = "1.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3b6c34f6f4b9e8f76274a9b309838d670b3bb69b4be6756394de54718aa2ca0a"
+dependencies = [
+ "aws-credential-types",
+ "aws-http",
+ "aws-runtime",
+ "aws-smithy-async",
+ "aws-smithy-http",
+ "aws-smithy-json",
+ "aws-smithy-runtime",
+ "aws-smithy-runtime-api",
+ "aws-smithy-types",
+ "aws-types",
+ "bytes",
+ "http 0.2.9",
+ "regex",
+ "tracing",
+]
+
 [[package]]
 name = "aws-sdk-kinesis"
 version = "1.3.0"
@@ -1713,9 +1637,9 @@ dependencies = [
 
 [[package]]
 name = "aws-smithy-runtime-api"
-version = "1.6.2"
+version = "1.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4179bd8a1c943e1aceb46c5b9fc014a561bd6c35a2153e816ba29076ee49d245"
+checksum = "30819352ed0a04ecf6a2f3477e344d2d1ba33d43e0f09ad9047c12e0d923616f"
 dependencies = [
  "aws-smithy-async",
  "aws-smithy-types",
@@ -1734,7 +1658,7 @@ version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cfe321a6b21f5d8eabd0ade9c55d3d0335f3c3157fc2b3e87f05f34b539e4df5"
 dependencies = [
- "base64-simd 0.8.0",
+ "base64-simd",
  "bytes",
  "bytes-utils",
  "futures-core",
@@ -1970,22 +1894,13 @@ version = "0.22.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9475866fec1451be56a3c2400fd081ff546538961565ccb5b7142cbd22bc7a51"
 
-[[package]]
-name = "base64-simd"
-version = "0.7.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "781dd20c3aff0bd194fe7d2a977dd92f21c173891f3a03b677359e5fa457e5d5"
-dependencies = [
- "simd-abstraction",
-]
-
 [[package]]
 name = "base64-simd"
 version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195"
 dependencies = [
- "outref 0.5.1",
+ "outref",
  "vsimd",
 ]
 
@@ -2024,15 +1939,6 @@ version = "0.23.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5f40afb3abbf90895dda3ddbc6d8734d24215130a22d646067690f5e318f81bc"
 
-[[package]]
-name = "better_scoped_tls"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "794edcc9b3fb07bb4aecaa11f093fd45663b4feadb782d68303a2268bc2701de"
-dependencies = [
- "scoped-tls",
-]
-
 [[package]]
 name = "bigdecimal"
 version = "0.3.1"
@@ -2098,7 +2004,7 @@ version = "0.69.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0"
 dependencies = [
- "bitflags 2.5.0",
+ "bitflags 2.6.0",
  "cexpr",
  "clang-sys",
  "itertools 0.12.1",
@@ -2112,7 +2018,7 @@ dependencies = [
  "rustc-hash",
  "shlex",
  "syn 2.0.66",
- "which 4.4.2",
+ "which",
 ]
 
 [[package]]
@@ -2138,9 +2044,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
 
 [[package]]
 name = "bitflags"
-version = "2.5.0"
+version = "2.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1"
+checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de"
 dependencies = [
  "serde",
 ]
@@ -2427,12 +2333,6 @@ version = "1.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "981520c98f422fcc584dc1a95c334e6953900b9106bc47a9839b81790009eb21"
 
-[[package]]
-name = "cache_control"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1bf2a5fb3207c12b5d208ebc145f967fea5cac41a021c37417ccc31ba40f39ee"
-
 [[package]]
 name = "camino"
 version = "1.1.6"
@@ -2789,17 +2689,6 @@ dependencies = [
  "cc",
 ]
 
-[[package]]
-name = "cmac"
-version = "0.7.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8543454e3c3f5126effff9cd44d562af4e31fb8ce1cc0d3dcd8f084515dbc1aa"
-dependencies = [
- "cipher",
- "dbl",
- "digest",
-]
-
 [[package]]
 name = "cmake"
 version = "0.1.50"
@@ -3005,12 +2894,6 @@ dependencies = [
  "unicode-segmentation",
 ]
 
-[[package]]
-name = "cooked-waker"
-version = "5.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "147be55d677052dabc6b22252d5dd0fd4c29c8c27aa4f2fbef0f94aa003b406f"
-
 [[package]]
 name = "core-foundation"
 version = "0.9.3"
@@ -3342,7 +3225,7 @@ version = "0.27.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f476fe445d41c9e991fd07515a6f463074b782242ccf4a5b7b1d1012e70824df"
 dependencies = [
- "bitflags 2.5.0",
+ "bitflags 2.6.0",
  "crossterm_winapi",
  "libc",
  "parking_lot 0.12.1",
@@ -3441,16 +3324,15 @@ dependencies = [
 
 [[package]]
 name = "curve25519-dalek"
-version = "4.1.2"
+version = "4.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0a677b8922c94e01bdbb12126b0bc852f00447528dee1782229af9c720c3f348"
+checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be"
 dependencies = [
  "cfg-if",
  "cpufeatures",
  "curve25519-dalek-derive",
  "digest",
  "fiat-crypto",
- "platforms",
  "rustc_version 0.4.0",
  "subtle",
  "zeroize",
@@ -3895,22 +3777,12 @@ dependencies = [
  "syn 1.0.109",
 ]
 
-[[package]]
-name = "dbl"
-version = "0.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bd2735a791158376708f9347fe8faba9667589d82427ef3aed6794a8981de3d9"
-dependencies = [
- "generic-array",
-]
-
 [[package]]
 name = "debugid"
 version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bef552e6f588e446098f6ba40d89ac146c8c7b64aade83c051ee00bb5d2bc18d"
 dependencies = [
- "serde",
  "uuid",
 ]
 
@@ -3984,426 +3856,82 @@ dependencies = [
 ]
 
 [[package]]
-name = "deno_ast"
-version = "0.34.2"
+name = "der"
+version = "0.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "58d986a1df3f1538ffa04162b5c5f00b856121391b860dc003bde2a6a741e878"
+checksum = "f1a467a65c5e759bce6e65eaf91cc29f466cdc57cb65777bd646872a8a1fd4de"
 dependencies = [
- "anyhow",
- "base64 0.21.7",
- "deno_media_type",
- "deno_terminal",
- "dprint-swc-ext",
- "once_cell",
- "percent-encoding",
- "serde",
- "swc_atoms",
- "swc_common",
- "swc_config",
- "swc_config_macro",
- "swc_ecma_ast",
- "swc_ecma_codegen",
- "swc_ecma_codegen_macros",
- "swc_ecma_loader",
- "swc_ecma_parser",
- "swc_ecma_transforms_base",
- "swc_ecma_transforms_classes",
- "swc_ecma_transforms_macros",
- "swc_ecma_transforms_proposal",
- "swc_ecma_transforms_react",
- "swc_ecma_transforms_typescript",
- "swc_ecma_utils",
- "swc_ecma_visit",
- "swc_eq_ignore_macros",
- "swc_macros_common",
- "swc_visit",
- "swc_visit_macros",
- "text_lines",
- "unicode-width",
- "url",
+ "const-oid",
+ "zeroize",
 ]
 
 [[package]]
-name = "deno_console"
-version = "0.144.0"
+name = "der"
+version = "0.7.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "372a00d0d0c42c3f5a451037e0ce327fa284144112c3109d32b50b40a2ef9532"
+checksum = "fffa369a668c8af7dbf8b5e56c9f744fbd399949ed171606040001947de40b1c"
 dependencies = [
- "deno_core",
+ "const-oid",
+ "pem-rfc7468",
+ "zeroize",
 ]
 
 [[package]]
-name = "deno_core"
-version = "0.272.0"
-source = "git+https://github.com/bakjos/deno_core?rev=9b241c6#9b241c6aa536fd64add55b34b9532efc2ced9b7c"
+name = "deranged"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0f32d04922c60427da6f9fef14d042d9edddef64cb9d4ce0d64d0685fbeb1fd3"
 dependencies = [
- "anyhow",
- "bincode 1.3.3",
- "bit-set",
- "bit-vec",
- "bytes",
- "cooked-waker",
- "deno_core_icudata",
- "deno_ops",
- "deno_unsync",
- "futures",
- "libc",
- "log",
- "memoffset",
- "parking_lot 0.12.1",
- "pin-project",
+ "powerfmt",
  "serde",
- "serde_json",
- "serde_v8",
- "smallvec",
- "sourcemap 7.1.1",
- "static_assertions",
- "tokio",
- "url",
- "v8",
 ]
 
 [[package]]
-name = "deno_core_icudata"
-version = "0.0.73"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a13951ea98c0a4c372f162d669193b4c9d991512de9f2381dd161027f34b26b1"
-
-[[package]]
-name = "deno_crypto"
-version = "0.158.0"
+name = "derivative"
+version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b26b7ac283934ee9c1ebb0b2328e4edda1487dd147d444c886993a96edf74b79"
+checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b"
 dependencies = [
- "aes",
- "aes-gcm",
- "aes-kw",
- "base64 0.21.7",
- "cbc",
- "const-oid",
- "ctr",
- "curve25519-dalek",
- "deno_core",
- "deno_web",
- "elliptic-curve 0.13.8",
- "num-traits",
- "once_cell",
- "p256 0.13.2",
- "p384",
- "p521",
- "rand",
- "ring 0.17.5",
- "rsa",
- "serde",
- "serde_bytes",
- "sha1",
- "sha2",
- "signature 2.2.0",
- "spki 0.7.2",
- "tokio",
- "uuid",
- "x25519-dalek",
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
 ]
 
 [[package]]
-name = "deno_fetch"
-version = "0.168.0"
-source = "git+https://github.com/bakjos/deno?rev=787a232#787a232c51f1ea390a2aff2b622016217d854328"
+name = "derive_builder"
+version = "0.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8d67778784b508018359cbc8696edb3db78160bab2c2a28ba7f56ef6932997f8"
 dependencies = [
- "bytes",
- "data-url",
- "deno_core",
- "deno_tls",
- "dyn-clone",
- "http 1.1.0",
- "pin-project",
- "reqwest 0.12.4",
- "serde",
- "serde_json",
- "tokio",
- "tokio-util",
+ "derive_builder_macro 0.12.0",
 ]
 
 [[package]]
-name = "deno_http"
-version = "0.141.0"
-source = "git+https://github.com/bakjos/deno?rev=787a232#787a232c51f1ea390a2aff2b622016217d854328"
+name = "derive_builder"
+version = "0.20.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0350b5cb0331628a5916d6c5c0b72e97393b8b6b03b47a9284f4e7f5a405ffd7"
 dependencies = [
- "async-compression",
- "async-trait",
- "base64 0.21.7",
- "brotli 3.5.0",
- "bytes",
- "cache_control",
- "deno_core",
- "deno_net",
- "deno_websocket",
- "flate2",
- "http 1.1.0",
- "httparse",
- "hyper 0.14.27",
- "hyper 1.1.0",
- "hyper-util",
- "itertools 0.10.5",
- "memmem",
- "mime",
- "once_cell",
- "percent-encoding",
- "phf",
- "pin-project",
- "ring 0.17.5",
- "scopeguard",
- "serde",
- "smallvec",
- "thiserror",
- "tokio",
- "tokio-util",
+ "derive_builder_macro 0.20.0",
 ]
 
 [[package]]
-name = "deno_io"
-version = "0.54.0"
+name = "derive_builder_core"
+version = "0.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3200f90cc9d232fd864cfcf25c535ab114fe824d9a38161e89418676360915e4"
+checksum = "c11bdc11a0c47bc7d37d582b5285da6849c96681023680b906673c5707af7b0f"
 dependencies = [
- "async-trait",
- "deno_core",
- "filetime",
- "fs3",
- "once_cell",
- "os_pipe",
- "rand",
- "tokio",
- "winapi",
+ "darling 0.14.4",
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
 ]
 
 [[package]]
-name = "deno_media_type"
-version = "0.1.3"
+name = "derive_builder_core"
+version = "0.20.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "edf9879493856d1622be70f396b0b0d3e519538dd6501b7c609ecbaa7e2194d2"
-dependencies = [
- "data-url",
- "serde",
- "url",
-]
-
-[[package]]
-name = "deno_native_certs"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f4785d0bdc13819b665b71e4fb7e119d859568471e4c245ec5610857e70c9345"
-dependencies = [
- "dlopen2",
- "dlopen2_derive",
- "once_cell",
- "rustls-native-certs 0.6.3",
- "rustls-pemfile 1.0.4",
-]
-
-[[package]]
-name = "deno_net"
-version = "0.136.0"
-source = "git+https://github.com/bakjos/deno?rev=787a232#787a232c51f1ea390a2aff2b622016217d854328"
-dependencies = [
- "deno_core",
- "deno_tls",
- "enum-as-inner 0.5.1",
- "log",
- "pin-project",
- "rustls-tokio-stream",
- "serde",
- "socket2 0.5.6",
- "tokio",
- "trust-dns-proto 0.22.0",
- "trust-dns-resolver 0.22.0",
-]
-
-[[package]]
-name = "deno_ops"
-version = "0.148.0"
-source = "git+https://github.com/bakjos/deno_core?rev=9b241c6#9b241c6aa536fd64add55b34b9532efc2ced9b7c"
-dependencies = [
- "proc-macro-rules",
- "proc-macro2",
- "quote",
- "strum 0.25.0",
- "strum_macros 0.25.3",
- "syn 2.0.66",
- "thiserror",
-]
-
-[[package]]
-name = "deno_terminal"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7e6337d4e7f375f8b986409a76fbeecfa4bd8a1343e63355729ae4befa058eaf"
-dependencies = [
- "once_cell",
- "termcolor",
-]
-
-[[package]]
-name = "deno_tls"
-version = "0.131.0"
-source = "git+https://github.com/bakjos/deno?rev=787a232#787a232c51f1ea390a2aff2b622016217d854328"
-dependencies = [
- "deno_core",
- "deno_native_certs",
- "once_cell",
- "rustls 0.22.4",
- "rustls-pemfile 1.0.4",
- "rustls-tokio-stream",
- "rustls-webpki 0.101.7",
- "serde",
- "webpki-roots 0.25.2",
-]
-
-[[package]]
-name = "deno_unsync"
-version = "0.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "30dff7e03584dbae188dae96a0f1876740054809b2ad0cf7c9fc5d361f20e739"
-dependencies = [
- "tokio",
-]
-
-[[package]]
-name = "deno_url"
-version = "0.144.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e172f44cffc771f0050d1befb12523d0420b6a44983d0c74d572609f4d3e02e3"
-dependencies = [
- "deno_core",
- "serde",
- "urlpattern",
-]
-
-[[package]]
-name = "deno_web"
-version = "0.175.0"
-source = "git+https://github.com/bakjos/deno?rev=787a232#787a232c51f1ea390a2aff2b622016217d854328"
-dependencies = [
- "async-trait",
- "base64-simd 0.8.0",
- "bytes",
- "deno_core",
- "encoding_rs",
- "flate2",
- "futures",
- "serde",
- "tokio",
- "uuid",
- "windows-sys 0.48.0",
-]
-
-[[package]]
-name = "deno_webidl"
-version = "0.144.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1c0484ffd8b436e926538fc0db4bc588832800b6681e554cebb3ae21f346c84e"
-dependencies = [
- "deno_core",
-]
-
-[[package]]
-name = "deno_websocket"
-version = "0.149.0"
-source = "git+https://github.com/bakjos/deno?rev=787a232#787a232c51f1ea390a2aff2b622016217d854328"
-dependencies = [
- "bytes",
- "deno_core",
- "deno_net",
- "deno_tls",
- "fastwebsockets",
- "h2 0.4.4",
- "http 1.1.0",
- "http-body-util",
- "hyper 1.1.0",
- "hyper-util",
- "once_cell",
- "rustls-tokio-stream",
- "serde",
- "tokio",
-]
-
-[[package]]
-name = "der"
-version = "0.6.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f1a467a65c5e759bce6e65eaf91cc29f466cdc57cb65777bd646872a8a1fd4de"
-dependencies = [
- "const-oid",
- "zeroize",
-]
-
-[[package]]
-name = "der"
-version = "0.7.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fffa369a668c8af7dbf8b5e56c9f744fbd399949ed171606040001947de40b1c"
-dependencies = [
- "const-oid",
- "pem-rfc7468",
- "zeroize",
-]
-
-[[package]]
-name = "deranged"
-version = "0.3.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0f32d04922c60427da6f9fef14d042d9edddef64cb9d4ce0d64d0685fbeb1fd3"
-dependencies = [
- "powerfmt",
- "serde",
-]
-
-[[package]]
-name = "derivative"
-version = "2.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 1.0.109",
-]
-
-[[package]]
-name = "derive_builder"
-version = "0.12.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8d67778784b508018359cbc8696edb3db78160bab2c2a28ba7f56ef6932997f8"
-dependencies = [
- "derive_builder_macro 0.12.0",
-]
-
-[[package]]
-name = "derive_builder"
-version = "0.20.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0350b5cb0331628a5916d6c5c0b72e97393b8b6b03b47a9284f4e7f5a405ffd7"
-dependencies = [
- "derive_builder_macro 0.20.0",
-]
-
-[[package]]
-name = "derive_builder_core"
-version = "0.12.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c11bdc11a0c47bc7d37d582b5285da6849c96681023680b906673c5707af7b0f"
-dependencies = [
- "darling 0.14.4",
- "proc-macro2",
- "quote",
- "syn 1.0.109",
-]
-
-[[package]]
-name = "derive_builder_core"
-version = "0.20.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d48cda787f839151732d396ac69e3473923d54312c070ee21e9effcaa8ca0b1d"
+checksum = "d48cda787f839151732d396ac69e3473923d54312c070ee21e9effcaa8ca0b1d"
 dependencies = [
  "darling 0.20.9",
  "proc-macro2",
@@ -4550,29 +4078,6 @@ version = "1.0.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "86e3bdc80eee6e16b2b6b0f87fbc98c04bee3455e35174c0de1a125d0688c632"
 
-[[package]]
-name = "dlopen2"
-version = "0.6.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6bc2c7ed06fd72a8513ded8d0d2f6fd2655a85d6885c48cae8625d80faf28c03"
-dependencies = [
- "dlopen2_derive",
- "libc",
- "once_cell",
- "winapi",
-]
-
-[[package]]
-name = "dlopen2_derive"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f2b99bf03862d7f545ebc28ddd33a665b50865f4dfd84031a393823879bd4c54"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.66",
-]
-
 [[package]]
 name = "dlv-list"
 version = "0.5.2"
@@ -4606,21 +4111,6 @@ version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9ea835d29036a4087793836fa931b08837ad5e957da9e23886b29586fb9b6650"
 
-[[package]]
-name = "dprint-swc-ext"
-version = "0.15.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5bad772f9e49af3a613fcddf1671d1e2e877e0a6d94f2b7162bfea4ac8140bee"
-dependencies = [
- "num-bigint",
- "rustc-hash",
- "swc_atoms",
- "swc_common",
- "swc_ecma_ast",
- "swc_ecma_parser",
- "text_lines",
-]
-
 [[package]]
 name = "duct"
 version = "0.13.6"
@@ -4730,9 +4220,9 @@ dependencies = [
 
 [[package]]
 name = "either"
-version = "1.12.0"
+version = "1.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3dca9240753cf90908d7e4aac30f630662b02aebaa1b58a3cadabdb23385b58b"
+checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0"
 dependencies = [
  "serde",
 ]
@@ -4784,70 +4274,6 @@ version = "0.3.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f"
 
-[[package]]
-name = "encoding"
-version = "0.2.33"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6b0d943856b990d12d3b55b359144ff341533e516d94098b1d3fc1ac666d36ec"
-dependencies = [
- "encoding-index-japanese",
- "encoding-index-korean",
- "encoding-index-simpchinese",
- "encoding-index-singlebyte",
- "encoding-index-tradchinese",
-]
-
-[[package]]
-name = "encoding-index-japanese"
-version = "1.20141219.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "04e8b2ff42e9a05335dbf8b5c6f7567e5591d0d916ccef4e0b1710d32a0d0c91"
-dependencies = [
- "encoding_index_tests",
-]
-
-[[package]]
-name = "encoding-index-korean"
-version = "1.20141219.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4dc33fb8e6bcba213fe2f14275f0963fd16f0a02c878e3095ecfdf5bee529d81"
-dependencies = [
- "encoding_index_tests",
-]
-
-[[package]]
-name = "encoding-index-simpchinese"
-version = "1.20141219.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d87a7194909b9118fc707194baa434a4e3b0fb6a5a757c73c3adb07aa25031f7"
-dependencies = [
- "encoding_index_tests",
-]
-
-[[package]]
-name = "encoding-index-singlebyte"
-version = "1.20141219.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3351d5acffb224af9ca265f435b859c7c01537c0849754d3db3fdf2bfe2ae84a"
-dependencies = [
- "encoding_index_tests",
-]
-
-[[package]]
-name = "encoding-index-tradchinese"
-version = "1.20141219.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fd0e20d5688ce3cab59eb3ef3a2083a5c77bf496cb798dc6fcdb75f323890c18"
-dependencies = [
- "encoding_index_tests",
-]
-
-[[package]]
-name = "encoding_index_tests"
-version = "0.1.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a246d82be1c9d791c5dfde9a2bd045fc3cbba3fa2b11ad558f27d01712f00569"
-
 [[package]]
 name = "encoding_rs"
 version = "0.8.33"
@@ -4869,18 +4295,6 @@ dependencies = [
  "syn 1.0.109",
 ]
 
-[[package]]
-name = "enum-as-inner"
-version = "0.5.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c9720bba047d567ffc8a3cba48bf19126600e249ab7f128e9233e6376976a116"
-dependencies = [
- "heck 0.4.1",
- "proc-macro2",
- "quote",
- "syn 1.0.109",
-]
-
 [[package]]
 name = "enum-as-inner"
 version = "0.6.0"
@@ -5022,8 +4436,9 @@ dependencies = [
 
 [[package]]
 name = "etcd-client"
-version = "0.12.1"
-source = "git+https://github.com/risingwavelabs/etcd-client.git?rev=4e84d40#4e84d40a84b35718d814cc2afccc9274c9d78e1e"
+version = "0.12.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4ae697f3928e8c89ae6f4dcf788059f49fd01a76dc53e63628f5a33881f5715e"
 dependencies = [
  "http 0.2.9",
  "prost 0.12.1",
@@ -5104,32 +4519,10 @@ dependencies = [
 ]
 
 [[package]]
-name = "failure"
-version = "0.1.8"
+name = "fallible-iterator"
+version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d32e9bd16cc02eae7db7ef620b392808b89f6a5e16bb3497d159c6b92a0f4f86"
-dependencies = [
- "backtrace",
- "failure_derive",
-]
-
-[[package]]
-name = "failure_derive"
-version = "0.1.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 1.0.109",
- "synstructure",
-]
-
-[[package]]
-name = "fallible-iterator"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7"
+checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7"
 
 [[package]]
 name = "fallible-iterator"
@@ -5172,25 +4565,6 @@ version = "2.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5"
 
-[[package]]
-name = "fastwebsockets"
-version = "0.6.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f63dd7b57f9b33b1741fa631c9522eb35d43e96dcca4a6a91d5e4ca7c93acdc1"
-dependencies = [
- "base64 0.21.7",
- "http-body-util",
- "hyper 1.1.0",
- "hyper-util",
- "pin-project",
- "rand",
- "sha1",
- "simdutf8",
- "thiserror",
- "tokio",
- "utf-8",
-]
-
 [[package]]
 name = "fd-lock"
 version = "4.0.1"
@@ -5237,18 +4611,6 @@ dependencies = [
  "bitflags 1.3.2",
 ]
 
-[[package]]
-name = "filetime"
-version = "0.2.23"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1ee447700ac8aa0b2f2bd7bc4462ad686ba06baa6727ac149a2d6277f0d240fd"
-dependencies = [
- "cfg-if",
- "libc",
- "redox_syscall 0.4.1",
- "windows-sys 0.52.0",
-]
-
 [[package]]
 name = "findshlibs"
 version = "0.10.2"
@@ -5409,9 +4771,9 @@ dependencies = [
 
 [[package]]
 name = "foyer"
-version = "0.10.0"
+version = "0.10.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cbfceb28a004d50b6110fd012db1d52d318d15f721dbb5b65eda261b821c1baa"
+checksum = "1b5f358e9b9492a9e9af905934ef4736a97a001fa19232f4a29f42974cf7a24c"
 dependencies = [
  "ahash 0.8.11",
  "anyhow",
@@ -5427,9 +4789,9 @@ dependencies = [
 
 [[package]]
 name = "foyer-common"
-version = "0.8.0"
+version = "0.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6ce1b903e1e142eafe8cd3183087fcd7ed6452e5dc8dfb5356a607ec2aa1c869"
+checksum = "e109f1fd012cc2f0785db5711c73c388fca2d2ac6e2aabb4eaa3a8af98b4dcdb"
 dependencies = [
  "bytes",
  "cfg-if",
@@ -5449,9 +4811,9 @@ dependencies = [
 
 [[package]]
 name = "foyer-intrusive"
-version = "0.8.0"
+version = "0.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2f7883ac7a8a69115f5bd84072a0b98d4ba72feacddf9040c217a5012bb352ff"
+checksum = "2e7ad91ad11f2c94bb4b3b88acd447f3145224415d776f163d2b06a9ed24e63a"
 dependencies = [
  "foyer-common",
  "itertools 0.13.0",
@@ -5459,12 +4821,12 @@ dependencies = [
 
 [[package]]
 name = "foyer-memory"
-version = "0.6.0"
+version = "0.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9fd7b0ce80867803c6b197db20cc1a49fcecc9d5070d2fb829660ec19acf9e72"
+checksum = "29273589433f89d61347b196754bfa22357420d1adbf815d354c23512106a194"
 dependencies = [
  "ahash 0.8.11",
- "bitflags 2.5.0",
+ "bitflags 2.6.0",
  "cmsketch",
  "foyer-common",
  "foyer-intrusive",
@@ -5482,9 +4844,9 @@ dependencies = [
 
 [[package]]
 name = "foyer-storage"
-version = "0.9.0"
+version = "0.9.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4205359a7ea9d05ff7525f0e55a5257a593c89b4e22ebfa07e06a84c2e163202"
+checksum = "79c20580e456ea337e4bdefa9fba7ff19b42cd87432bda3ef3579ef9e1057c7e"
 dependencies = [
  "ahash 0.8.11",
  "allocator-api2",
@@ -5492,7 +4854,7 @@ dependencies = [
  "array-util",
  "async-channel 2.2.1",
  "bincode 1.3.3",
- "bitflags 2.5.0",
+ "bitflags 2.6.0",
  "bytes",
  "either",
  "foyer-common",
@@ -5520,17 +4882,6 @@ version = "2.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6c2141d6d6c8512188a7891b4b01590a45f6dac67afb4f255c4124dbb86d4eaa"
 
-[[package]]
-name = "from_variant"
-version = "0.1.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3a0b11eeb173ce52f84ebd943d42e58813a2ebb78a6a3ff0a243b71c5199cd7b"
-dependencies = [
- "proc-macro2",
- "swc_macros_common",
- "syn 2.0.66",
-]
-
 [[package]]
 name = "frunk"
 version = "0.4.2"
@@ -5613,33 +4964,12 @@ dependencies = [
  "winapi",
 ]
 
-[[package]]
-name = "fs3"
-version = "0.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fb17cf6ed704f72485332f6ab65257460c4f9f3083934cf402bf9f5b3b600a90"
-dependencies = [
- "libc",
- "rustc_version 0.2.3",
- "winapi",
-]
-
 [[package]]
 name = "fs_extra"
 version = "1.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
 
-[[package]]
-name = "fslock"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "04412b8935272e3a9bae6f48c7bfff74c2911f60525404edfdd28e49884c3bfb"
-dependencies = [
- "libc",
- "winapi",
-]
-
 [[package]]
 name = "function_name"
 version = "0.3.0"
@@ -5678,9 +5008,9 @@ dependencies = [
 
 [[package]]
 name = "futures-async-stream"
-version = "0.2.9"
+version = "0.2.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "379790776b0d953337df4ab7ecc51936c66ea112484cad7912907b1d34253ebf"
+checksum = "6cce57e88ba9fe4953f476112b2c8e315a2da07725a14dc091ac3e5b6e4cca72"
 dependencies = [
  "futures-async-stream-macro",
  "futures-core",
@@ -5689,9 +5019,9 @@ dependencies = [
 
 [[package]]
 name = "futures-async-stream-macro"
-version = "0.2.9"
+version = "0.2.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5df2c13d48c8cb8a3ec093ede6f0f4482f327d7bb781120c5fb483ef0f17e758"
+checksum = "5ac45ed0bddbd110eb68862768a194f88700f5b91c39931d2f432fab67a16d08"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -5830,7 +5160,7 @@ version = "0.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "27d12c0aed7f1e24276a241aadc4cb8ea9f83000f34bc062b7cc2d51e3b0fabd"
 dependencies = [
- "bitflags 2.5.0",
+ "bitflags 2.6.0",
  "debugid",
  "fxhash",
  "serde",
@@ -6123,15 +5453,6 @@ dependencies = [
  "subtle",
 ]
 
-[[package]]
-name = "gzip-header"
-version = "1.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "95cc527b92e6029a62960ad99aa8a6660faa4555fe5f731aab13aa6a921795a2"
-dependencies = [
- "crc32fast",
-]
-
 [[package]]
 name = "h2"
 version = "0.3.26"
@@ -6333,20 +5654,6 @@ dependencies = [
  "windows 0.52.0",
 ]
 
-[[package]]
-name = "hstr"
-version = "0.2.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b0f5356d62012374578cd3a5c013d6586de3efbca3b53379fc1edfbb95c9db14"
-dependencies = [
- "hashbrown 0.14.3",
- "new_debug_unreachable",
- "once_cell",
- "phf",
- "rustc-hash",
- "triomphe",
-]
-
 [[package]]
 name = "http"
 version = "0.2.9"
@@ -6605,7 +5912,7 @@ dependencies = [
 [[package]]
 name = "iceberg"
 version = "0.2.0"
-source = "git+https://github.com/risingwavelabs/iceberg-rust.git?rev=0c6e133e6f4655ff9ce4ad57b577dc7f692dd902#0c6e133e6f4655ff9ce4ad57b577dc7f692dd902"
+source = "git+https://github.com/risingwavelabs/iceberg-rust.git?rev=24bd5869f2779a8b9786b5a6e1f9723844f5a82c#24bd5869f2779a8b9786b5a6e1f9723844f5a82c"
 dependencies = [
  "anyhow",
  "apache-avro 0.17.0",
@@ -6652,7 +5959,7 @@ dependencies = [
 [[package]]
 name = "iceberg-catalog-rest"
 version = "0.2.0"
-source = "git+https://github.com/risingwavelabs/iceberg-rust.git?rev=0c6e133e6f4655ff9ce4ad57b577dc7f692dd902#0c6e133e6f4655ff9ce4ad57b577dc7f692dd902"
+source = "git+https://github.com/risingwavelabs/iceberg-rust.git?rev=24bd5869f2779a8b9786b5a6e1f9723844f5a82c#24bd5869f2779a8b9786b5a6e1f9723844f5a82c"
 dependencies = [
  "async-trait",
  "chrono",
@@ -6758,12 +6065,6 @@ dependencies = [
  "unicode-normalization",
 ]
 
-[[package]]
-name = "if_chain"
-version = "1.0.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cb56e1aa765b4b4f3aadfab769793b7087bb03a4ea4920644a6d238e2df5b9ed"
-
 [[package]]
 name = "indexmap"
 version = "1.9.3"
@@ -6856,7 +6157,7 @@ version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bd05e4e63529f3c9c5f5c668c398217f72756ffe48c85266b49692c55accd1f7"
 dependencies = [
- "bitflags 2.5.0",
+ "bitflags 2.6.0",
  "crossterm 0.25.0",
  "dyn-clone",
  "fuzzy-matcher",
@@ -6927,18 +6228,6 @@ version = "2.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "28b29a3cd74f0f4598934efe3aeba42bae0eb4680554128851ebbecb02af14e6"
 
-[[package]]
-name = "is-macro"
-version = "0.3.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "59a85abdc13717906baccb5a1e435556ce0df215f242892f721dff62bf25288f"
-dependencies = [
- "Inflector",
- "proc-macro2",
- "quote",
- "syn 2.0.66",
-]
-
 [[package]]
 name = "is-terminal"
 version = "0.4.9"
@@ -7280,7 +6569,7 @@ version = "0.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8"
 dependencies = [
- "bitflags 2.5.0",
+ "bitflags 2.6.0",
  "libc",
  "redox_syscall 0.4.1",
 ]
@@ -7710,12 +6999,6 @@ dependencies = [
  "libc",
 ]
 
-[[package]]
-name = "memmem"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a64a92489e2744ce060c349162be1c5f33c6969234104dbd99ddb5feb08b8c15"
-
 [[package]]
 name = "memoffset"
 version = "0.9.0"
@@ -8049,7 +7332,7 @@ dependencies = [
  "base64 0.21.7",
  "bigdecimal 0.4.5",
  "bindgen",
- "bitflags 2.5.0",
+ "bitflags 2.6.0",
  "bitvec",
  "btoi",
  "byteorder",
@@ -8104,12 +7387,6 @@ dependencies = [
  "tempfile",
 ]
 
-[[package]]
-name = "new_debug_unreachable"
-version = "1.0.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086"
-
 [[package]]
 name = "newline-converter"
 version = "0.3.0"
@@ -8147,7 +7424,7 @@ version = "0.29.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46"
 dependencies = [
- "bitflags 2.5.0",
+ "bitflags 2.6.0",
  "cfg-if",
  "cfg_aliases",
  "libc",
@@ -8256,7 +7533,6 @@ checksum = "c165a9ab64cf766f73521c0dd2cfdff64f488b8f0b3e621face3462d3db536d7"
 dependencies = [
  "num-integer",
  "num-traits",
- "rand",
  "serde",
 ]
 
@@ -8286,6 +7562,12 @@ dependencies = [
  "num-traits",
 ]
 
+[[package]]
+name = "num-conv"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9"
+
 [[package]]
 name = "num-format"
 version = "0.4.4"
@@ -8539,11 +7821,11 @@ dependencies = [
 
 [[package]]
 name = "openssl"
-version = "0.10.64"
+version = "0.10.66"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f"
+checksum = "9529f4786b70a3e8c61e11179af17ab6188ad8d0ded78c5529441ed39d4bd9c1"
 dependencies = [
- "bitflags 2.5.0",
+ "bitflags 2.6.0",
  "cfg-if",
  "foreign-types",
  "libc",
@@ -8569,14 +7851,24 @@ version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"
 
+[[package]]
+name = "openssl-src"
+version = "300.3.1+3.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7259953d42a81bf137fbbd73bd30a8e1914d6dce43c2b90ed575783a22608b91"
+dependencies = [
+ "cc",
+]
+
 [[package]]
 name = "openssl-sys"
-version = "0.9.101"
+version = "0.9.103"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dda2b0f344e78efc2facf7d195d098df0dd72151b26ab98da807afc26c198dff"
+checksum = "7f9e8deee91df40a943c71b917e5874b951d32a802526c85721ce3b776c929d6"
 dependencies = [
  "cc",
  "libc",
+ "openssl-src",
  "pkg-config",
  "vcpkg",
 ]
@@ -8653,6 +7945,20 @@ dependencies = [
  "tokio-stream",
 ]
 
+[[package]]
+name = "opentls"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6f561874f8d6ecfb674fc08863414040c93cc90c0b6963fe679895fab8b65560"
+dependencies = [
+ "futures-util",
+ "log",
+ "openssl",
+ "openssl-probe",
+ "openssl-sys",
+ "url",
+]
+
 [[package]]
 name = "ordered-float"
 version = "2.10.0"
@@ -8744,12 +8050,6 @@ dependencies = [
  "syn 2.0.66",
 ]
 
-[[package]]
-name = "outref"
-version = "0.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7f222829ae9293e33a9f5e9f440c6760a3d450a64affe1846486b140db81c1f4"
-
 [[package]]
 name = "outref"
 version = "0.5.1"
@@ -8803,20 +8103,6 @@ dependencies = [
  "sha2",
 ]
 
-[[package]]
-name = "p521"
-version = "0.13.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0fc9e2161f1f215afdfce23677034ae137bbd45016a880c2eb3ba8eb95f085b2"
-dependencies = [
- "base16ct 0.2.0",
- "ecdsa 0.16.9",
- "elliptic-curve 0.13.8",
- "primeorder",
- "rand_core",
- "sha2",
-]
-
 [[package]]
 name = "panic-message"
 version = "0.3.0"
@@ -8914,38 +8200,6 @@ dependencies = [
  "zstd 0.13.0",
 ]
 
-[[package]]
-name = "parquet"
-version = "50.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "547b92ebf0c1177e3892f44c8f79757ee62e678d564a9834189725f2c5b7a750"
-dependencies = [
- "ahash 0.8.11",
- "arrow-array 50.0.0",
- "arrow-buffer 50.0.0",
- "arrow-cast 50.0.0",
- "arrow-data 50.0.0",
- "arrow-ipc 50.0.0",
- "arrow-schema 50.0.0",
- "arrow-select 50.0.0",
- "base64 0.21.7",
- "brotli 3.5.0",
- "bytes",
- "chrono",
- "flate2",
- "half 2.3.1",
- "hashbrown 0.14.3",
- "lz4_flex",
- "num",
- "num-bigint",
- "paste",
- "seq-macro",
- "snap",
- "thrift",
- "twox-hash",
- "zstd 0.13.0",
-]
-
 [[package]]
 name = "parquet"
 version = "52.0.0"
@@ -9039,12 +8293,6 @@ dependencies = [
  "once_cell",
 ]
 
-[[package]]
-name = "pathdiff"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8835116a5c179084a830efb3adc117ab007512b535bc1a21c991d3b32a6b44dd"
-
 [[package]]
 name = "pbjson"
 version = "0.6.0"
@@ -9180,7 +8428,6 @@ version = "0.11.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc"
 dependencies = [
- "phf_macros",
  "phf_shared",
 ]
 
@@ -9204,19 +8451,6 @@ dependencies = [
  "rand",
 ]
 
-[[package]]
-name = "phf_macros"
-version = "0.11.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3444646e286606587e49f3bcf1679b8cef1dc2c5ecc29ddacaffc305180d464b"
-dependencies = [
- "phf_generator",
- "phf_shared",
- "proc-macro2",
- "quote",
- "syn 2.0.66",
-]
-
 [[package]]
 name = "phf_shared"
 version = "0.11.2"
@@ -9313,22 +8547,6 @@ version = "0.3.27"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964"
 
-[[package]]
-name = "platforms"
-version = "3.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4503fa043bf02cee09a9582e9554b4c6403b2ef55e4612e96561d294419429f8"
-
-[[package]]
-name = "plotlib"
-version = "0.5.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9462104f987d8d0f6625f0c7764f1c8b890bd1dc8584d8293e031f25c5a0d242"
-dependencies = [
- "failure",
- "svg",
-]
-
 [[package]]
 name = "plotters"
 version = "0.3.5"
@@ -9357,17 +8575,6 @@ dependencies = [
  "plotters-backend",
 ]
 
-[[package]]
-name = "pmutil"
-version = "0.6.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "52a40bc70c2c58040d2d8b167ba9a5ff59fc9dab7ad44771cfde3dcfde7a09c6"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.66",
-]
-
 [[package]]
 name = "polling"
 version = "2.8.0"
@@ -9656,29 +8863,6 @@ version = "0.5.20+deprecated"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068"
 
-[[package]]
-name = "proc-macro-rules"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "07c277e4e643ef00c1233393c673f655e3672cf7eb3ba08a00bdd0ea59139b5f"
-dependencies = [
- "proc-macro-rules-macros",
- "proc-macro2",
- "syn 2.0.66",
-]
-
-[[package]]
-name = "proc-macro-rules-macros"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "207fffb0fe655d1d47f6af98cc2793405e85929bdbc420d685554ff07be27ac7"
-dependencies = [
- "once_cell",
- "proc-macro2",
- "quote",
- "syn 2.0.66",
-]
-
 [[package]]
 name = "proc-macro2"
 version = "1.0.85"
@@ -9707,7 +8891,7 @@ version = "0.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "731e0d9356b0c25f16f33b5be79b1c57b562f141ebfcdb0ad8ac2c13a24293b4"
 dependencies = [
- "bitflags 2.5.0",
+ "bitflags 2.6.0",
  "hex",
  "lazy_static",
  "procfs-core",
@@ -9720,7 +8904,7 @@ version = "0.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2d3554923a69f4ce04c4a754260c338f505ce22642d3830e049a399fc2059a29"
 dependencies = [
- "bitflags 2.5.0",
+ "bitflags 2.6.0",
  "hex",
 ]
 
@@ -9794,7 +8978,7 @@ dependencies = [
  "regex",
  "syn 1.0.109",
  "tempfile",
- "which 4.4.2",
+ "which",
 ]
 
 [[package]]
@@ -9816,7 +9000,7 @@ dependencies = [
  "regex",
  "syn 2.0.66",
  "tempfile",
- "which 4.4.2",
+ "which",
 ]
 
 [[package]]
@@ -10000,7 +9184,7 @@ dependencies = [
  "indoc",
  "libc",
  "memoffset",
- "parking_lot 0.12.1",
+ "parking_lot 0.11.2",
  "portable-atomic",
  "pyo3-build-config",
  "pyo3-ffi",
@@ -10453,7 +9637,6 @@ version = "0.12.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "566cafdd92868e0939d3fb961bd0dc25fcfaaed179291093b3d43e6b3150ea10"
 dependencies = [
- "async-compression",
  "base64 0.22.0",
  "bytes",
  "encoding_rs",
@@ -10488,7 +9671,6 @@ dependencies = [
  "tokio",
  "tokio-native-tls",
  "tokio-rustls 0.25.0",
- "tokio-socks",
  "tokio-util",
  "tower-service",
  "url",
@@ -10694,6 +9876,7 @@ dependencies = [
  "memcomparable",
  "opendal",
  "parking_lot 0.12.1",
+ "parquet 52.0.0",
  "paste",
  "prometheus",
  "prost 0.12.1",
@@ -10747,7 +9930,7 @@ dependencies = [
  "nix 0.29.0",
  "opentelemetry",
  "parking_lot 0.12.1",
- "plotlib",
+ "plotters",
  "prometheus",
  "rand",
  "risingwave_common",
@@ -10841,7 +10024,7 @@ dependencies = [
  "async-trait",
  "auto_enums",
  "auto_impl",
- "bitflags 2.5.0",
+ "bitflags 2.6.0",
  "byteorder",
  "bytes",
  "chrono",
@@ -10899,6 +10082,7 @@ dependencies = [
  "risingwave_common_estimate_size",
  "risingwave_common_metrics",
  "risingwave_common_proc_macro",
+ "risingwave_common_secret",
  "risingwave_error",
  "risingwave_license",
  "risingwave_pb",
@@ -11013,17 +10197,33 @@ dependencies = [
 ]
 
 [[package]]
-name = "risingwave_common_service"
+name = "risingwave_common_secret"
 version = "1.11.0-alpha"
 dependencies = [
- "async-trait",
- "axum 0.7.4",
- "futures",
- "hyper 0.14.27",
- "madsim-tokio",
- "madsim-tonic",
- "prometheus",
- "risingwave_common",
+ "aes-gcm",
+ "anyhow",
+ "bincode 1.3.3",
+ "parking_lot 0.12.1",
+ "prost 0.12.1",
+ "risingwave_pb",
+ "serde",
+ "thiserror",
+ "thiserror-ext",
+ "tracing",
+]
+
+[[package]]
+name = "risingwave_common_service"
+version = "1.11.0-alpha"
+dependencies = [
+ "async-trait",
+ "axum 0.7.4",
+ "futures",
+ "hyper 0.14.27",
+ "madsim-tokio",
+ "madsim-tonic",
+ "prometheus",
+ "risingwave_common",
  "risingwave_pb",
  "risingwave_rpc_client",
  "thiserror",
@@ -11069,6 +10269,7 @@ dependencies = [
  "async-trait",
  "await-tree",
  "clap",
+ "jsonbb",
  "madsim-tokio",
  "madsim-tonic",
  "parking_lot 0.12.1",
@@ -11127,6 +10328,7 @@ dependencies = [
  "tokio-stream",
  "tower",
  "tracing",
+ "uuid",
  "workspace-hack",
 ]
 
@@ -11153,6 +10355,7 @@ dependencies = [
  "aws-credential-types",
  "aws-msk-iam-sasl-signer",
  "aws-sdk-dynamodb",
+ "aws-sdk-glue",
  "aws-sdk-kinesis",
  "aws-sdk-s3",
  "aws-smithy-http",
@@ -11203,7 +10406,7 @@ dependencies = [
  "opendal",
  "openssl",
  "parking_lot 0.12.1",
- "parquet 50.0.0",
+ "parquet 52.0.0",
  "paste",
  "pg_bigdecimal",
  "postgres-openssl",
@@ -11275,6 +10478,8 @@ dependencies = [
  "apache-avro 0.16.0",
  "chrono",
  "easy-ext",
+ "expect-test",
+ "hex",
  "itertools 0.12.1",
  "jsonbb",
  "jsonschema-transpiler",
@@ -11441,11 +10646,11 @@ dependencies = [
  "arrow-schema 52.0.0",
  "arrow-udf-flight",
  "arrow-udf-js",
- "arrow-udf-js-deno",
  "arrow-udf-python",
  "arrow-udf-wasm",
  "async-trait",
  "auto_enums",
+ "bytes",
  "chrono",
  "chrono-tz 0.9.0",
  "criterion",
@@ -11513,6 +10718,7 @@ dependencies = [
  "base64 0.22.0",
  "bk-tree",
  "bytes",
+ "chrono",
  "clap",
  "downcast-rs",
  "dyn-clone",
@@ -11526,6 +10732,7 @@ dependencies = [
  "futures",
  "futures-async-stream",
  "iana-time-zone",
+ "iceberg",
  "icelake",
  "itertools 0.12.1",
  "jsonbb",
@@ -11750,7 +10957,6 @@ dependencies = [
 name = "risingwave_meta"
 version = "1.11.0-alpha"
 dependencies = [
- "aes-siv",
  "anyhow",
  "arc-swap",
  "assert_matches",
@@ -11775,6 +10981,7 @@ dependencies = [
  "hex",
  "hyper 0.14.27",
  "itertools 0.12.1",
+ "jsonbb",
  "madsim-etcd-client",
  "madsim-tokio",
  "madsim-tonic",
@@ -11848,6 +11055,7 @@ name = "risingwave_meta_model_migration"
 version = "1.11.0-alpha"
 dependencies = [
  "async-std",
+ "sea-orm",
  "sea-orm-migration",
  "serde",
  "serde_json",
@@ -11873,8 +11081,10 @@ version = "1.11.0-alpha"
 dependencies = [
  "anyhow",
  "clap",
+ "educe",
  "either",
  "futures",
+ "hex",
  "itertools 0.12.1",
  "madsim-etcd-client",
  "madsim-tokio",
@@ -11910,6 +11120,7 @@ dependencies = [
  "itertools 0.12.1",
  "madsim-tokio",
  "madsim-tonic",
+ "prost 0.12.1",
  "rand",
  "regex",
  "risingwave_common",
@@ -12305,6 +11516,7 @@ dependencies = [
  "governor",
  "hytra",
  "itertools 0.12.1",
+ "jsonbb",
  "local_stats_alloc",
  "lru 0.7.6",
  "madsim-tokio",
@@ -12632,7 +11844,7 @@ version = "0.38.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6ea3e1a662af26cd7a3ba09c0297a31af215563ecf42817c98df621387f4e949"
 dependencies = [
- "bitflags 2.5.0",
+ "bitflags 2.6.0",
  "errno",
  "itoa",
  "libc",
@@ -12641,18 +11853,6 @@ dependencies = [
  "windows-sys 0.52.0",
 ]
 
-[[package]]
-name = "rustls"
-version = "0.20.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1b80e3dec595989ea8510028f30c408a4630db12c9cbb8de34203b89d6577e99"
-dependencies = [
- "log",
- "ring 0.16.20",
- "sct",
- "webpki",
-]
-
 [[package]]
 name = "rustls"
 version = "0.21.11"
@@ -12745,18 +11945,6 @@ version = "1.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d"
 
-[[package]]
-name = "rustls-tokio-stream"
-version = "0.2.18"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "696a389edb0b54b9bb888c8318404d9a0c0b9091fb03ca3d9c64731511db03f6"
-dependencies = [
- "futures",
- "rustls 0.22.4",
- "socket2 0.5.6",
- "tokio",
-]
-
 [[package]]
 name = "rustls-webpki"
 version = "0.101.7"
@@ -12854,12 +12042,6 @@ version = "1.0.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f"
 
-[[package]]
-name = "ryu-js"
-version = "1.0.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ad97d4ce1560a5e27cec89519dc8300d1aa6035b099821261c651486a19e44d5"
-
 [[package]]
 name = "salsa20"
 version = "0.10.2"
@@ -12926,12 +12108,6 @@ dependencies = [
  "hashbrown 0.13.2",
 ]
 
-[[package]]
-name = "scoped-tls"
-version = "1.0.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294"
-
 [[package]]
 name = "scopeguard"
 version = "1.2.0"
@@ -13391,19 +12567,6 @@ dependencies = [
  "serde",
 ]
 
-[[package]]
-name = "serde_v8"
-version = "0.181.0"
-source = "git+https://github.com/bakjos/deno_core?rev=9b241c6#9b241c6aa536fd64add55b34b9532efc2ced9b7c"
-dependencies = [
- "bytes",
- "num-bigint",
- "serde",
- "smallvec",
- "thiserror",
- "v8",
-]
-
 [[package]]
 name = "serde_with"
 version = "1.14.0"
@@ -13644,15 +12807,6 @@ dependencies = [
  "rand_core",
 ]
 
-[[package]]
-name = "simd-abstraction"
-version = "0.7.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9cadb29c57caadc51ff8346233b5cec1d240b68ce55cf1afc764818791876987"
-dependencies = [
- "outref 0.1.0",
-]
-
 [[package]]
 name = "simd-json"
 version = "0.13.3"
@@ -13677,9 +12831,9 @@ checksum = "f27f6278552951f1f2b8cf9da965d10969b2efdea95a6ec47987ab46edfe263a"
 
 [[package]]
 name = "similar"
-version = "2.5.0"
+version = "2.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fa42c91313f1d05da9b26f267f931cf178d4aba455b4c4622dd7355eb80c6640"
+checksum = "1de1d4f81173b03af4c0cbed3c898f6bff5b870e4a7f5d6f4057d62a7a4b686e"
 
 [[package]]
 name = "simple_asn1"
@@ -13769,17 +12923,6 @@ dependencies = [
  "serde",
 ]
 
-[[package]]
-name = "smartstring"
-version = "1.0.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3fb72c633efbaa2dd666986505016c32c3044395ceaf881518399d2f4127ee29"
-dependencies = [
- "autocfg",
- "static_assertions",
- "version_check",
-]
-
 [[package]]
 name = "snafu"
 version = "0.7.5"
@@ -13828,39 +12971,6 @@ dependencies = [
  "windows-sys 0.52.0",
 ]
 
-[[package]]
-name = "sourcemap"
-version = "6.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e4cbf65ca7dc576cf50e21f8d0712d96d4fcfd797389744b7b222a85cdf5bd90"
-dependencies = [
- "data-encoding",
- "debugid",
- "if_chain",
- "rustc_version 0.2.3",
- "serde",
- "serde_json",
- "unicode-id",
- "url",
-]
-
-[[package]]
-name = "sourcemap"
-version = "7.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e7768edd06c02535e0d50653968f46e1e0d3aa54742190d35dd9466f59de9c71"
-dependencies = [
- "base64-simd 0.7.0",
- "data-encoding",
- "debugid",
- "if_chain",
- "rustc_version 0.2.3",
- "serde",
- "serde_json",
- "unicode-id-start",
- "url",
-]
-
 [[package]]
 name = "speedate"
 version = "0.14.0"
@@ -14088,7 +13198,7 @@ dependencies = [
  "atoi",
  "base64 0.21.7",
  "bigdecimal 0.3.1",
- "bitflags 2.5.0",
+ "bitflags 2.6.0",
  "byteorder",
  "bytes",
  "chrono",
@@ -14135,7 +13245,7 @@ dependencies = [
  "atoi",
  "base64 0.21.7",
  "bigdecimal 0.3.1",
- "bitflags 2.5.0",
+ "bitflags 2.6.0",
  "byteorder",
  "chrono",
  "crc",
@@ -14227,18 +13337,6 @@ version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9091b6114800a5f2141aee1d1b9d6ca3592ac062dc5decb3764ec5895a47b4eb"
 
-[[package]]
-name = "string_enum"
-version = "0.4.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1b650ea2087d32854a0f20b837fc56ec987a1cb4f758c9757e1171ee9812da63"
-dependencies = [
- "proc-macro2",
- "quote",
- "swc_macros_common",
- "syn 2.0.66",
-]
-
 [[package]]
 name = "stringprep"
 version = "0.1.4"
@@ -14264,454 +13362,97 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
 
 [[package]]
 name = "structmeta"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2e1575d8d40908d70f6fd05537266b90ae71b15dbbe7a8b7dffa2b759306d329"
-dependencies = [
- "proc-macro2",
- "quote",
- "structmeta-derive",
- "syn 2.0.66",
-]
-
-[[package]]
-name = "structmeta-derive"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "152a0b65a590ff6c3da95cabe2353ee04e6167c896b28e3b14478c2636c922fc"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.66",
-]
-
-[[package]]
-name = "strum"
-version = "0.25.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125"
-dependencies = [
- "strum_macros 0.25.3",
-]
-
-[[package]]
-name = "strum"
-version = "0.26.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5d8cec3501a5194c432b2b7976db6b7d10ec95c253208b45f83f7136aa985e29"
-dependencies = [
- "strum_macros 0.26.4",
-]
-
-[[package]]
-name = "strum_macros"
-version = "0.25.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "23dc1fa9ac9c169a78ba62f0b841814b7abae11bdd047b9c58f893439e309ea0"
-dependencies = [
- "heck 0.4.1",
- "proc-macro2",
- "quote",
- "rustversion",
- "syn 2.0.66",
-]
-
-[[package]]
-name = "strum_macros"
-version = "0.26.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be"
-dependencies = [
- "heck 0.5.0",
- "proc-macro2",
- "quote",
- "rustversion",
- "syn 2.0.66",
-]
-
-[[package]]
-name = "subprocess"
-version = "0.2.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c2e86926081dda636c546d8c5e641661049d7562a68f5488be4a1f7f66f6086"
-dependencies = [
- "libc",
- "winapi",
-]
-
-[[package]]
-name = "subst"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ca1318e5d6716d6541696727c88d9b8dfc8cfe6afd6908e186546fd4af7f5b98"
-dependencies = [
- "memchr",
- "unicode-width",
-]
-
-[[package]]
-name = "subtle"
-version = "2.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc"
-
-[[package]]
-name = "svg"
-version = "0.7.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3685c82a045a6af0c488f0550b0f52b4c77d2a52b0ca8aba719f9d268fa96965"
-
-[[package]]
-name = "swc_atoms"
-version = "0.6.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7d538eaaa6f085161d088a04cf0a3a5a52c5a7f2b3bd9b83f73f058b0ed357c0"
-dependencies = [
- "hstr",
- "once_cell",
- "rustc-hash",
- "serde",
-]
-
-[[package]]
-name = "swc_cached"
-version = "0.3.19"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "630c761c74ac8021490b78578cc2223aa4a568241e26505c27bf0e4fd4ad8ec2"
-dependencies = [
- "ahash 0.8.11",
- "anyhow",
- "dashmap",
- "once_cell",
- "regex",
- "serde",
-]
-
-[[package]]
-name = "swc_common"
-version = "0.33.18"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c85e8b15d0fb87691e27c8f3cf953748db3ccd2a39e165d6d5275a48fb0d29e3"
-dependencies = [
- "ast_node",
- "better_scoped_tls",
- "cfg-if",
- "either",
- "from_variant",
- "new_debug_unreachable",
- "num-bigint",
- "once_cell",
- "rustc-hash",
- "serde",
- "siphasher",
- "sourcemap 6.4.1",
- "swc_atoms",
- "swc_eq_ignore_macros",
- "swc_visit",
- "tracing",
- "unicode-width",
- "url",
-]
-
-[[package]]
-name = "swc_config"
-version = "0.1.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ce837c5eae1cb200a310940de989fd9b3d12ed62d7752bc69b39ef8aa775ec04"
-dependencies = [
- "anyhow",
- "indexmap 2.2.6",
- "serde",
- "serde_json",
- "swc_cached",
- "swc_config_macro",
-]
-
-[[package]]
-name = "swc_config_macro"
-version = "0.1.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8b2574f75082322a27d990116cd2a24de52945fc94172b24ca0b3e9e2a6ceb6b"
-dependencies = [
- "proc-macro2",
- "quote",
- "swc_macros_common",
- "syn 2.0.66",
-]
-
-[[package]]
-name = "swc_ecma_ast"
-version = "0.112.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "36226eb87bfd2f5620bde04f149a4b869ab34e78496d60cb0d8eb9da765d0732"
-dependencies = [
- "bitflags 2.5.0",
- "is-macro",
- "num-bigint",
- "phf",
- "scoped-tls",
- "serde",
- "string_enum",
- "swc_atoms",
- "swc_common",
- "unicode-id",
-]
-
-[[package]]
-name = "swc_ecma_codegen"
-version = "0.148.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5ba8669ab28bb5d1e65c1e8690257c026745ac368e0101c2c6544d4a03afc95e"
-dependencies = [
- "memchr",
- "num-bigint",
- "once_cell",
- "rustc-hash",
- "serde",
- "sourcemap 6.4.1",
- "swc_atoms",
- "swc_common",
- "swc_ecma_ast",
- "swc_ecma_codegen_macros",
- "tracing",
-]
-
-[[package]]
-name = "swc_ecma_codegen_macros"
-version = "0.7.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "394b8239424b339a12012ceb18726ed0244fce6bf6345053cb9320b2791dcaa5"
-dependencies = [
- "proc-macro2",
- "quote",
- "swc_macros_common",
- "syn 2.0.66",
-]
-
-[[package]]
-name = "swc_ecma_loader"
-version = "0.45.20"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d0058cf970880f5382effe43eb2b727a73ba09ae41922fa140c2c3fa6ca9b2d1"
-dependencies = [
- "anyhow",
- "pathdiff",
- "serde",
- "swc_atoms",
- "swc_common",
- "tracing",
-]
-
-[[package]]
-name = "swc_ecma_parser"
-version = "0.143.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "20823cac99a9adbd4c03fb5e126aaccbf92446afedad99252a0e1fc76e2ffc43"
-dependencies = [
- "either",
- "new_debug_unreachable",
- "num-bigint",
- "num-traits",
- "phf",
- "serde",
- "smallvec",
- "smartstring",
- "stacker",
- "swc_atoms",
- "swc_common",
- "swc_ecma_ast",
- "tracing",
- "typed-arena",
-]
-
-[[package]]
-name = "swc_ecma_transforms_base"
-version = "0.137.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "66539401f619730b26d380a120b91b499f80cbdd9bb15d00aa73bc3a4d4cc394"
-dependencies = [
- "better_scoped_tls",
- "bitflags 2.5.0",
- "indexmap 2.2.6",
- "once_cell",
- "phf",
- "rustc-hash",
- "serde",
- "smallvec",
- "swc_atoms",
- "swc_common",
- "swc_ecma_ast",
- "swc_ecma_parser",
- "swc_ecma_utils",
- "swc_ecma_visit",
- "tracing",
-]
-
-[[package]]
-name = "swc_ecma_transforms_classes"
-version = "0.126.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ebf9048e687b746d2bbe6149601c3eedd819fef08d7657e5fddcef99b22febba"
-dependencies = [
- "swc_atoms",
- "swc_common",
- "swc_ecma_ast",
- "swc_ecma_transforms_base",
- "swc_ecma_utils",
- "swc_ecma_visit",
-]
-
-[[package]]
-name = "swc_ecma_transforms_macros"
-version = "0.5.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "17e309b88f337da54ef7fe4c5b99c2c522927071f797ee6c9fb8b6bf2d100481"
-dependencies = [
- "proc-macro2",
- "quote",
- "swc_macros_common",
- "syn 2.0.66",
-]
-
-[[package]]
-name = "swc_ecma_transforms_proposal"
-version = "0.171.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "35f0a72ee781aa9208836046fd2c12e483f5515898858511b68863290cb97b45"
-dependencies = [
- "either",
- "rustc-hash",
- "serde",
- "smallvec",
- "swc_atoms",
- "swc_common",
- "swc_ecma_ast",
- "swc_ecma_transforms_base",
- "swc_ecma_transforms_classes",
- "swc_ecma_transforms_macros",
- "swc_ecma_utils",
- "swc_ecma_visit",
-]
-
-[[package]]
-name = "swc_ecma_transforms_react"
-version = "0.183.10"
+version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f0ec75c1194365abe4d44d94e58f918ec853469ecd39733b381a089cfdcdee1a"
+checksum = "2e1575d8d40908d70f6fd05537266b90ae71b15dbbe7a8b7dffa2b759306d329"
 dependencies = [
- "base64 0.21.7",
- "dashmap",
- "indexmap 2.2.6",
- "once_cell",
- "serde",
- "sha-1",
- "string_enum",
- "swc_atoms",
- "swc_common",
- "swc_config",
- "swc_ecma_ast",
- "swc_ecma_parser",
- "swc_ecma_transforms_base",
- "swc_ecma_transforms_macros",
- "swc_ecma_utils",
- "swc_ecma_visit",
+ "proc-macro2",
+ "quote",
+ "structmeta-derive",
+ "syn 2.0.66",
 ]
 
 [[package]]
-name = "swc_ecma_transforms_typescript"
-version = "0.188.10"
+name = "structmeta-derive"
+version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fec5e95a9c840eb13562884123eaa627cb6e05e0461c94a2ce69ae7e70313010"
+checksum = "152a0b65a590ff6c3da95cabe2353ee04e6167c896b28e3b14478c2636c922fc"
 dependencies = [
- "ryu-js",
- "serde",
- "swc_atoms",
- "swc_common",
- "swc_ecma_ast",
- "swc_ecma_transforms_base",
- "swc_ecma_transforms_react",
- "swc_ecma_utils",
- "swc_ecma_visit",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.66",
 ]
 
 [[package]]
-name = "swc_ecma_utils"
-version = "0.127.7"
+name = "strum"
+version = "0.25.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "14482e455df85486d68a51533a31645d511e56df93a35cadf0eabbe7abe96b98"
+checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125"
 dependencies = [
- "indexmap 2.2.6",
- "num_cpus",
- "once_cell",
- "rustc-hash",
- "swc_atoms",
- "swc_common",
- "swc_ecma_ast",
- "swc_ecma_visit",
- "tracing",
- "unicode-id",
+ "strum_macros 0.25.3",
 ]
 
 [[package]]
-name = "swc_ecma_visit"
-version = "0.98.4"
+name = "strum"
+version = "0.26.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "df0127694c36d656ea9eab5c170cdd8ab398246ae2a335de26961c913a4aca47"
+checksum = "5d8cec3501a5194c432b2b7976db6b7d10ec95c253208b45f83f7136aa985e29"
 dependencies = [
- "num-bigint",
- "swc_atoms",
- "swc_common",
- "swc_ecma_ast",
- "swc_visit",
- "tracing",
+ "strum_macros 0.26.4",
 ]
 
 [[package]]
-name = "swc_eq_ignore_macros"
-version = "0.1.3"
+name = "strum_macros"
+version = "0.25.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "695a1d8b461033d32429b5befbf0ad4d7a2c4d6ba9cd5ba4e0645c615839e8e4"
+checksum = "23dc1fa9ac9c169a78ba62f0b841814b7abae11bdd047b9c58f893439e309ea0"
 dependencies = [
+ "heck 0.4.1",
  "proc-macro2",
  "quote",
+ "rustversion",
  "syn 2.0.66",
 ]
 
 [[package]]
-name = "swc_macros_common"
-version = "0.3.9"
+name = "strum_macros"
+version = "0.26.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "50176cfc1cbc8bb22f41c6fe9d1ec53fbe057001219b5954961b8ad0f336fce9"
+checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be"
 dependencies = [
+ "heck 0.5.0",
  "proc-macro2",
  "quote",
+ "rustversion",
  "syn 2.0.66",
 ]
 
 [[package]]
-name = "swc_visit"
-version = "0.5.9"
+name = "subprocess"
+version = "0.2.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "358e246dedeb4ae8efacebcce1360dc2f9b6c0b4c1ad8b737cc60f5b6633691a"
+checksum = "0c2e86926081dda636c546d8c5e641661049d7562a68f5488be4a1f7f66f6086"
 dependencies = [
- "either",
- "swc_visit_macros",
+ "libc",
+ "winapi",
 ]
 
 [[package]]
-name = "swc_visit_macros"
-version = "0.5.10"
+name = "subst"
+version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fbbbb9d77d5112f90ed7ea00477135b16c4370c872b93a0b63b766e8710650ad"
+checksum = "ca1318e5d6716d6541696727c88d9b8dfc8cfe6afd6908e186546fd4af7f5b98"
 dependencies = [
- "Inflector",
- "pmutil",
- "proc-macro2",
- "quote",
- "swc_macros_common",
- "syn 2.0.66",
+ "memchr",
+ "unicode-width",
 ]
 
+[[package]]
+name = "subtle"
+version = "2.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc"
+
 [[package]]
 name = "symbolic-common"
 version = "12.4.0"
@@ -14796,18 +13537,6 @@ version = "0.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160"
 
-[[package]]
-name = "synstructure"
-version = "0.12.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 1.0.109",
- "unicode-xid",
-]
-
 [[package]]
 name = "sysinfo"
 version = "0.30.0"
@@ -14849,7 +13578,7 @@ version = "0.27.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b858526d22750088a9b3cf2e3c2aacebd5377f13adeec02860c30d09113010a6"
 dependencies = [
- "bitflags 2.5.0",
+ "bitflags 2.6.0",
  "cap-fs-ext",
  "cap-std",
  "fd-lock",
@@ -14910,15 +13639,6 @@ version = "0.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76"
 
-[[package]]
-name = "text_lines"
-version = "0.6.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7fd5828de7deaa782e1dd713006ae96b3bee32d3279b79eb67ecf8072c059bcf"
-dependencies = [
- "serde",
-]
-
 [[package]]
 name = "thiserror"
 version = "1.0.61"
@@ -15005,8 +13725,7 @@ dependencies = [
 [[package]]
 name = "tiberius"
 version = "0.12.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dc6e2bf3e4b5be181a2a2ceff4b9b12e2684010d436a6958bd564fbc8094d44d"
+source = "git+https://github.com/risingwavelabs/tiberius.git?rev=f834f2deeb9e2fb08afaf73865f330cf31a3876a#f834f2deeb9e2fb08afaf73865f330cf31a3876a"
 dependencies = [
  "async-trait",
  "asynchronous-codec",
@@ -15015,19 +13734,18 @@ dependencies = [
  "bytes",
  "chrono",
  "connection-string",
- "encoding",
+ "encoding_rs",
  "enumflags2",
  "futures-util",
  "num-traits",
  "once_cell",
+ "opentls",
  "pin-project-lite",
  "pretty-hex",
  "rust_decimal",
- "rustls-native-certs 0.6.3",
- "rustls-pemfile 1.0.4",
  "thiserror",
  "time",
- "tokio-rustls 0.23.4",
+ "tokio",
  "tokio-util",
  "tracing",
  "uuid",
@@ -15063,13 +13781,14 @@ dependencies = [
 
 [[package]]
 name = "time"
-version = "0.3.30"
+version = "0.3.36"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c4a34ab300f2dee6e562c10a046fc05e358b29f9bf92277f30c3c8d82275f6f5"
+checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885"
 dependencies = [
  "deranged",
  "itoa",
  "libc",
+ "num-conv",
  "num_threads",
  "powerfmt",
  "serde",
@@ -15085,10 +13804,11 @@ checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3"
 
 [[package]]
 name = "time-macros"
-version = "0.2.15"
+version = "0.2.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4ad70d68dba9e1f8aceda7aa6711965dfec1cac869f311a51bd08b3a2ccbce20"
+checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf"
 dependencies = [
+ "num-conv",
  "time-core",
 ]
 
@@ -15236,17 +13956,6 @@ dependencies = [
  "rand",
 ]
 
-[[package]]
-name = "tokio-rustls"
-version = "0.23.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59"
-dependencies = [
- "rustls 0.20.9",
- "tokio",
- "webpki",
-]
-
 [[package]]
 name = "tokio-rustls"
 version = "0.24.1"
@@ -15279,18 +13988,6 @@ dependencies = [
  "tokio",
 ]
 
-[[package]]
-name = "tokio-socks"
-version = "0.5.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "51165dfa029d2a65969413a6cc96f354b86b464498702f174a4efa13608fd8c0"
-dependencies = [
- "either",
- "futures-util",
- "thiserror",
- "tokio",
-]
-
 [[package]]
 name = "tokio-stream"
 version = "0.1.14"
@@ -15485,7 +14182,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5"
 dependencies = [
  "async-compression",
- "bitflags 2.5.0",
+ "bitflags 2.6.0",
  "bytes",
  "futures-core",
  "futures-util",
@@ -15669,10 +14366,6 @@ name = "triomphe"
 version = "0.1.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "859eb650cfee7434994602c3a68b25d77ad9e68c8a6cd491616ef86661382eb3"
-dependencies = [
- "serde",
- "stable_deref_trait",
-]
 
 [[package]]
 name = "triple_accel"
@@ -15705,32 +14398,6 @@ dependencies = [
  "url",
 ]
 
-[[package]]
-name = "trust-dns-proto"
-version = "0.22.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4f7f83d1e4a0e4358ac54c5c3681e5d7da5efc5a7a632c90bb6d6669ddd9bc26"
-dependencies = [
- "async-trait",
- "cfg-if",
- "data-encoding",
- "enum-as-inner 0.5.1",
- "futures-channel",
- "futures-io",
- "futures-util",
- "idna 0.2.3",
- "ipnet",
- "lazy_static",
- "rand",
- "serde",
- "smallvec",
- "thiserror",
- "tinyvec",
- "tokio",
- "tracing",
- "url",
-]
-
 [[package]]
 name = "trust-dns-proto"
 version = "0.23.2"
@@ -15776,27 +14443,6 @@ dependencies = [
  "trust-dns-proto 0.21.2",
 ]
 
-[[package]]
-name = "trust-dns-resolver"
-version = "0.22.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aff21aa4dcefb0a1afbfac26deb0adc93888c7d295fb63ab273ef276ba2b7cfe"
-dependencies = [
- "cfg-if",
- "futures-util",
- "ipconfig",
- "lazy_static",
- "lru-cache",
- "parking_lot 0.12.1",
- "resolv-conf",
- "serde",
- "smallvec",
- "thiserror",
- "tokio",
- "tracing",
- "trust-dns-proto 0.22.0",
-]
-
 [[package]]
 name = "trust-dns-resolver"
 version = "0.23.2"
@@ -15846,12 +14492,6 @@ dependencies = [
  "static_assertions",
 ]
 
-[[package]]
-name = "typed-arena"
-version = "2.0.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6af6ae20167a9ece4bcb41af5b80f8a1f1df981f6391189ce00fd257af04126a"
-
 [[package]]
 name = "typed-builder"
 version = "0.10.0"
@@ -15918,47 +14558,6 @@ dependencies = [
  "version_check",
 ]
 
-[[package]]
-name = "unic-char-property"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a8c57a407d9b6fa02b4795eb81c5b6652060a15a7903ea981f3d723e6c0be221"
-dependencies = [
- "unic-char-range",
-]
-
-[[package]]
-name = "unic-char-range"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0398022d5f700414f6b899e10b8348231abf9173fa93144cbc1a43b9793c1fbc"
-
-[[package]]
-name = "unic-common"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "80d7ff825a6a654ee85a63e80f92f054f904f21e7d12da4e22f9834a4aaa35bc"
-
-[[package]]
-name = "unic-ucd-ident"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e230a37c0381caa9219d67cf063aa3a375ffed5bf541a452db16e744bdab6987"
-dependencies = [
- "unic-char-property",
- "unic-char-range",
- "unic-ucd-version",
-]
-
-[[package]]
-name = "unic-ucd-version"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "96bd2f2237fe450fcd0a1d2f5f4e91711124f7857ba2e964247776ebeeb7b0c4"
-dependencies = [
- "unic-common",
-]
-
 [[package]]
 name = "unicase"
 version = "2.7.0"
@@ -15974,18 +14573,6 @@ version = "0.3.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460"
 
-[[package]]
-name = "unicode-id"
-version = "0.3.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b1b6def86329695390197b82c1e244a54a131ceb66c996f2088a3876e2ae083f"
-
-[[package]]
-name = "unicode-id-start"
-version = "1.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b8f73150333cb58412db36f2aca8f2875b013049705cc77b94ded70a1ab1f5da"
-
 [[package]]
 name = "unicode-ident"
 version = "1.0.11"
@@ -16077,25 +14664,6 @@ version = "2.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da"
 
-[[package]]
-name = "urlpattern"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f9bd5ff03aea02fa45b13a7980151fe45009af1980ba69f651ec367121a31609"
-dependencies = [
- "derive_more",
- "regex",
- "serde",
- "unic-ucd-ident",
- "url",
-]
-
-[[package]]
-name = "utf-8"
-version = "0.7.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9"
-
 [[package]]
 name = "utf8parse"
 version = "0.2.1"
@@ -16104,30 +14672,15 @@ checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a"
 
 [[package]]
 name = "uuid"
-version = "1.8.0"
+version = "1.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a183cf7feeba97b4dd1c0d46788634f6221d87fa961b305bed08c851829efcc0"
+checksum = "81dfa00651efa65069b0b6b651f4aaa31ba9e3c3ce0137aaad053604ee7e0314"
 dependencies = [
  "getrandom",
  "rand",
  "serde",
 ]
 
-[[package]]
-name = "v8"
-version = "0.89.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fe2197fbef82c98f7953d13568a961d4e1c663793b5caf3c74455a13918cdf33"
-dependencies = [
- "bitflags 2.5.0",
- "fslock",
- "gzip-header",
- "home",
- "miniz_oxide",
- "once_cell",
- "which 5.0.0",
-]
-
 [[package]]
 name = "valuable"
 version = "0.1.0"
@@ -16235,7 +14788,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "63255d85e10627b07325d7cf4e5fe5a40fa4ff183569a0a67931be26d50ede07"
 dependencies = [
  "anyhow",
- "bitflags 2.5.0",
+ "bitflags 2.6.0",
  "cap-fs-ext",
  "cap-rand",
  "cap-std",
@@ -16376,7 +14929,7 @@ version = "0.202.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d6998515d3cf3f8b980ef7c11b29a9b1017d4cf86b99ae93b546992df9931413"
 dependencies = [
- "bitflags 2.5.0",
+ "bitflags 2.6.0",
  "indexmap 2.2.6",
  "semver 1.0.18",
 ]
@@ -16721,16 +15274,6 @@ dependencies = [
  "wasm-bindgen",
 ]
 
-[[package]]
-name = "webpki"
-version = "0.22.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ed63aea5ce73d0ff405984102c42de94fc55a6b75765d621c65262469b3c9b53"
-dependencies = [
- "ring 0.17.5",
- "untrusted 0.9.0",
-]
-
 [[package]]
 name = "webpki-roots"
 version = "0.25.2"
@@ -16758,19 +15301,6 @@ dependencies = [
  "rustix 0.38.31",
 ]
 
-[[package]]
-name = "which"
-version = "5.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9bf3ea8596f3a0dd5980b46430f2058dfe2c36a27ccfbb1845d6fbfcd9ba6e14"
-dependencies = [
- "either",
- "home",
- "once_cell",
- "rustix 0.38.31",
- "windows-sys 0.48.0",
-]
-
 [[package]]
 name = "whoami"
 version = "1.5.0"
@@ -16796,7 +15326,7 @@ checksum = "1b6552dda951239e219c329e5a768393664e8d120c5e0818487ac2633f173b1f"
 dependencies = [
  "anyhow",
  "async-trait",
- "bitflags 2.5.0",
+ "bitflags 2.6.0",
  "thiserror",
  "tracing",
  "wasmtime",
@@ -17167,7 +15697,7 @@ version = "0.36.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f9643b83820c0cd246ecabe5fa454dd04ba4fa67996369466d0747472d337346"
 dependencies = [
- "bitflags 2.5.0",
+ "bitflags 2.6.0",
  "windows-sys 0.52.0",
 ]
 
@@ -17235,18 +15765,6 @@ dependencies = [
  "tap",
 ]
 
-[[package]]
-name = "x25519-dalek"
-version = "2.0.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c7e468321c81fb07fa7f4c636c3972b9100f0346e5b6a9f2bd0603a52f7ed277"
-dependencies = [
- "curve25519-dalek",
- "rand_core",
- "serde",
- "zeroize",
-]
-
 [[package]]
 name = "xmlparser"
 version = "0.13.5"
@@ -17351,20 +15869,6 @@ name = "zeroize"
 version = "1.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d"
-dependencies = [
- "zeroize_derive",
-]
-
-[[package]]
-name = "zeroize_derive"
-version = "1.4.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.66",
-]
 
 [[package]]
 name = "zstd"
diff --git a/Cargo.toml b/Cargo.toml
index 80f3f5af2e8c7..bbf28a2cd1b49 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -77,7 +77,7 @@ license = "Apache-2.0"
 repository = "https://github.com/risingwavelabs/risingwave"
 
 [workspace.dependencies]
-foyer = { version = "0.10.0", features = ["nightly", "mtrace"] }
+foyer = { version = "0.10.1", features = ["nightly", "mtrace"] }
 apache-avro = { git = "https://github.com/risingwavelabs/avro", rev = "25113ba88234a9ae23296e981d8302c290fdaa4b", features = [
     "snappy",
     "zstandard",
@@ -94,6 +94,7 @@ aws-config = { version = "1", default-features = false, features = [
 aws-credential-types = { version = "1", default-features = false, features = [
     "hardcoded-credentials",
 ] }
+aws-sdk-glue = "1"
 aws-sdk-kinesis = { version = "1", default-features = false, features = [
     "rt-tokio",
     "rustls",
@@ -141,8 +142,9 @@ arrow-buffer-iceberg = { package = "arrow-buffer", version = "52" }
 arrow-cast-iceberg = { package = "arrow-cast", version = "52" }
 # TODO
 # After apache/iceberg-rust#411 is merged, we move to the upstream version.
-iceberg = { git = "https://github.com/risingwavelabs/iceberg-rust.git", rev = "0c6e133e6f4655ff9ce4ad57b577dc7f692dd902" }
-iceberg-catalog-rest = { git = "https://github.com/risingwavelabs/iceberg-rust.git", rev = "0c6e133e6f4655ff9ce4ad57b577dc7f692dd902" }
+iceberg = { git = "https://github.com/risingwavelabs/iceberg-rust.git", rev = "24bd5869f2779a8b9786b5a6e1f9723844f5a82c" }
+iceberg-catalog-rest = { git = "https://github.com/risingwavelabs/iceberg-rust.git", rev = "24bd5869f2779a8b9786b5a6e1f9723844f5a82c" }
+opendal = "0.47"
 arrow-array = "50"
 arrow-arith = "50"
 arrow-cast = "50"
@@ -153,7 +155,6 @@ arrow-select = "50"
 arrow-ord = "50"
 arrow-row = "50"
 arrow-udf-js = "0.3.1"
-arrow-udf-js-deno = { git = "https://github.com/risingwavelabs/arrow-udf.git", rev = "fa36365" }
 arrow-udf-wasm = { version = "0.2.2", features = ["build"] }
 arrow-udf-python = "0.2"
 arrow-udf-flight = "0.1"
@@ -171,7 +172,7 @@ deltalake = { git = "https://github.com/risingwavelabs/delta-rs", rev = "5c2dccd
 itertools = "0.12.0"
 jsonbb = "0.1.4"
 lru = { git = "https://github.com/risingwavelabs/lru-rs.git", rev = "2682b85" }
-parquet = "50"
+parquet = { version = "52", features = ["async"] }
 thiserror-ext = "0.1.2"
 tikv-jemalloc-ctl = { git = "https://github.com/risingwavelabs/jemallocator.git", rev = "64a2d9" }
 tikv-jemallocator = { git = "https://github.com/risingwavelabs/jemallocator.git", features = [
@@ -246,11 +247,16 @@ rw_iter_util = { path = "src/utils/iter_util" }
 [workspace.lints.rust]
 # `forbid` will also prevent the misuse of `#[allow(unused)]`
 unused_must_use = "forbid"
-future_incompatible = "warn"
-nonstandard_style = "warn"
-rust_2018_idioms = "warn"
+future_incompatible = { level = "warn", priority = -1 }
+nonstandard_style = { level = "warn", priority = -1 }
+rust_2018_idioms = { level = "warn", priority = -1 }
 # Backward compatibility is not important for an application.
 async_fn_in_trait = "allow"
+unexpected_cfgs = { level = "warn", check-cfg = [
+    'cfg(madsim)',
+    'cfg(coverage)',
+    'cfg(dashboard_built)',
+] }
 
 [workspace.lints.clippy]
 uninlined_format_args = "allow"
@@ -280,17 +286,22 @@ private_intra_doc_links = "allow"
 # Explicit lints don't hurt, and sometimes rust-analyzer works better with explicit links.
 redundant_explicit_links = "allow"
 
+# Tweak built-in profiles and define custom profiles.
+# See `docs/dev/src/build-and-run/profiles.md` for detailed information.
 [profile.dev]
-lto = 'off'
+lto = "off"
 
 [profile.release]
 debug = "full"
+incremental = true
 split-debuginfo = "packed"
+lto = "off"
+
+[profile.production]
+inherits = "release"
+incremental = false
 lto = "thin"
 
-# The profile used for CI in main branch.
-# This profile inherits from the release profile, but turns on some checks and assertions for us to
-# better catch bugs in CI.
 [profile.ci-release]
 inherits = "release"
 incremental = false
@@ -300,9 +311,6 @@ split-debuginfo = "off"
 debug-assertions = true
 overflow-checks = true
 
-# The profile used for CI in pull requests.
-# External dependencies are built with optimization enabled, while crates in this workspace are built
-# with `dev` profile and full debug info. This is a trade-off between build time and e2e test time.
 [profile.ci-dev]
 inherits = "dev"
 incremental = false
@@ -316,14 +324,11 @@ opt-level = 3
 [profile.ci-dev.package."indextree"]
 opt-level = 3
 
-# The profile used for deterministic simulation tests in CI.
-# The simulator can only run single-threaded, so optimization is required to make the running time
-# reasonable. The optimization level is customized to speed up the build.
 [profile.ci-sim]
 inherits = "dev"
-opt-level = 2
 incremental = false
-debug = 1
+debug = "line-tables-only"
+opt-level = 2
 
 [patch.crates-io]
 # Patch third-party crates for deterministic simulation.
@@ -333,17 +338,6 @@ tokio-stream = { git = "https://github.com/madsim-rs/tokio.git", rev = "fe39bb8e
 tokio-retry = { git = "https://github.com/madsim-rs/rust-tokio-retry.git", rev = "95e2fd3" }
 tokio-postgres = { git = "https://github.com/madsim-rs/rust-postgres.git", rev = "ac00d88" }
 futures-timer = { git = "https://github.com/madsim-rs/futures-timer.git", rev = "05b33b4" }
-# patch: unlimit 4MB message size for grpc client
-etcd-client = { git = "https://github.com/risingwavelabs/etcd-client.git", rev = "4e84d40" }
-# patch to remove preserve_order from serde_json
-deno_core = { git = "https://github.com/bakjos/deno_core", rev = "9b241c6" }
-# patch to user reqwest 0.12.2
-deno_fetch = { git = "https://github.com/bakjos/deno", rev = "787a232" }
-deno_http = { git = "https://github.com/bakjos/deno", rev = "787a232" }
-deno_net = { git = "https://github.com/bakjos/deno", rev = "787a232" }
-deno_tls = { git = "https://github.com/bakjos/deno", rev = "787a232" }
-deno_web = { git = "https://github.com/bakjos/deno", rev = "787a232" }
-deno_websocket = { git = "https://github.com/bakjos/deno", rev = "787a232" }
 # patch to remove preserve_order from serde_json
 bson = { git = "https://github.com/risingwavelabs/bson-rust", rev = "e5175ec" }
 
diff --git a/Makefile.toml b/Makefile.toml
index a93e832f1859f..2fe13db1ca8cc 100644
--- a/Makefile.toml
+++ b/Makefile.toml
@@ -28,6 +28,8 @@ env_scripts = [
 
   set_env ENABLE_TELEMETRY "false"
   set_env RW_TELEMETRY_TYPE "test"
+  set_env RW_SECRET_STORE_PRIVATE_KEY_HEX "0123456789abcdef"
+  set_env RW_TEMP_SECRET_FILE_DIR "${PREFIX_SECRET}"
 
   is_sanitizer_enabled = get_env ENABLE_SANITIZER
   is_hdfs_backend = get_env ENABLE_HDFS
@@ -38,7 +40,6 @@ env_scripts = [
   is_external_udf_enabled = get_env ENABLE_EXTERNAL_UDF
   is_wasm_udf_enabled = get_env ENABLE_WASM_UDF
   is_js_udf_enabled = get_env ENABLE_JS_UDF
-  is_deno_udf_enabled = get_env ENABLE_DENO_UDF
   is_python_udf_enabled = get_env ENABLE_PYTHON_UDF
 
   if ${is_sanitizer_enabled}
@@ -77,11 +78,6 @@ env_scripts = [
     set_env RISINGWAVE_FEATURE_FLAGS "${flags} --features js-udf"
   end
 
-  if ${is_deno_udf_enabled}
-    flags = get_env RISINGWAVE_FEATURE_FLAGS
-    set_env RISINGWAVE_FEATURE_FLAGS "${flags} --features deno-udf"
-  end
-
   if ${is_python_udf_enabled}
     flags = get_env RISINGWAVE_FEATURE_FLAGS
     set_env RISINGWAVE_FEATURE_FLAGS "${flags} --features python-udf"
@@ -144,6 +140,7 @@ rm -rf "${PREFIX_DATA}"
 rm -rf "${PREFIX_LOG}"
 rm -rf "${PREFIX_CONFIG}"
 rm -rf "${PREFIX_PROFILING}"
+rm -rf "${PREFIX_SECRET}"
 '''
 
 [tasks.reset-rw]
@@ -372,7 +369,7 @@ ln -s "$(pwd)/target/${RISEDEV_BUILD_TARGET_DIR}${BUILD_MODE_DIR}/risingwave" "$
 
 [tasks.post-build-risingwave]
 category = "RiseDev - Build"
-description = "Copy RisngWave binaries to bin"
+description = "Copy RisingWave binaries to bin"
 condition = { env_set = ["ENABLE_BUILD_RUST"] }
 dependencies = [
     "link-all-in-one-binaries",
@@ -578,7 +575,7 @@ set -euo pipefail
 RC_ENV_FILE="${PREFIX_CONFIG}/risedev-env"
 
 if [ ! -f "${RC_ENV_FILE}" ]; then
-  echo "risedev-env file not found. Did you start cluster using $(tput setaf 4)\`./risedev d\`$(tput sgr0) or $(tput setaf 4)\`./risedev p\`$(tput sgr0)?"
+  echo "risedev-env file not found at ${RC_ENV_FILE}. Did you start cluster using $(tput setaf 4)\`./risedev d\`$(tput sgr0) or $(tput setaf 4)\`./risedev p\`$(tput sgr0)?"
   exit 1
 fi
 '''
@@ -710,10 +707,10 @@ if ! ${TMUX} ls &>/dev/null ; then
   exit 0
 fi
 
-# Kill other components with Ctrl+C/Ctrl+D
+# Kill tmux components with Ctrl+C
 ${TMUX} list-windows -F "#{window_name} #{pane_id}" \
 | awk '{ print $2 }' \
-| xargs -I {} ${TMUX} send-keys -t {} C-c C-d
+| xargs -I {} ${TMUX} send-keys -t {} C-c
 
 # Stop docker components
 containers=$(docker ps -a -q -f name=risedev- 2>/dev/null) || true
@@ -763,7 +760,7 @@ set -e
 for tool in cargo-llvm-cov cargo-nextest cargo-hakari cargo-sort cargo-make cargo-udeps
 do
   echo "install: $(tput setaf 4)$tool$(tput sgr0)"
-  cargo binstall -y --no-symlinks $tool
+  cargo binstall -y --locked --no-symlinks $tool
   echo
 done
 
@@ -1135,7 +1132,7 @@ fi
 private = true
 category = "RiseDev - Check"
 description = "Run cargo typos-cli check"
-install_crate = { min_version = "1.20.4", crate_name = "typos-cli", binary = "typos", test_arg = [
+install_crate = { min_version = "1.23.2", crate_name = "typos-cli", binary = "typos", test_arg = [
     "--help",
 ], install_command = "binstall" }
 script = """
@@ -1258,7 +1255,7 @@ echo If you still feel this is not enough, you may copy $(tput setaf 4)risedev$(
 [tasks.ci-start]
 category = "RiseDev - CI"
 dependencies = ["clean-data", "pre-start-dev"]
-command = "target/${BUILD_MODE_DIR}/risedev-dev"
+command = "target/debug/risedev-dev" # `risedev-dev` is always built in dev profile
 args = ["${@}"]
 description = "Clean data and start a full RisingWave dev cluster using risedev-dev"
 
diff --git a/README.md b/README.md
index 156b53060313d..bf50ae208a972 100644
--- a/README.md
+++ b/README.md
@@ -54,11 +54,18 @@
   </a>
 </div>
 
-RisingWave is a Postgres-compatible streaming database engineered to provide the <i><b>simplest</b></i> and <i><b>most cost-efficient</b></i> approach for <b>processing</b>, <b>analyzing</b>, and <b>managing</b> real-time event streaming data.
-
+RisingWave is a Postgres-compatible SQL engine engineered to provide the <i><b>simplest</b></i> and <i><b>most cost-efficient</b></i> approach for <b>processing</b>, <b>analyzing</b>, and <b>managing</b> real-time event streaming data.
 
 ![RisingWave](https://github.com/risingwavelabs/risingwave/assets/41638002/10c44404-f78b-43ce-bbd9-3646690acc59)
 
+## When to use RisingWave?
+RisingWave can ingest millions of events per second, continuously join live data streams with historical tables, and serve ad-hoc queries in real-time. Typical use cases include, but are not limited to:
+
+* **Streaming analytics**: Perform streaming analytics and build live dashboards with data freshness under one second, ideal for stock trading, sports betting, IoT monitoring, and more.
+* **Event-driven applications**: Develop monitoring and alerting applications for fraud detection, anomaly detection, and more.
+* **Real-time ETL pipelines**: Ingest data from different sources, perform enrichment queries, and deliver results to downstream systems.
+* **Feature stores**: Transform both batch and streaming data into ML features using the same codebase.
+
 
 ## Try it out in 60 seconds
 
@@ -126,16 +133,6 @@ RisingWave is fundamentally a database that **extends beyond basic streaming dat
 * Schema change
 * Processing of semi-structured data
 
-## In-production use cases
-Within your data stack, RisingWave can assist with:
-
-* Processing and transforming event streaming data in real time
-* Offloading event-driven queries (e.g., materialized views, triggers) from operational databases
-* Performing real-time ETL (Extract, Transform, Load)
-* Supporting real-time feature stores
-
-Read more at [use cases](https://risingwave.com/use-cases/). RisingWave is extensively utilized in real-time applications such as monitoring, alerting, dashboard reporting, machine learning, among others. It has already been adopted in fields such as financial trading, manufacturing, new media, logistics, gaming, and more. Check out [customer stories](https://risingwave.com/resources/?filter=customer-stories).
-
 ## Community
 
 Looking for help, discussions, collaboration opportunities, or a casual afternoon chat with our fellow engineers and community members? Join our [Slack workspace](https://risingwave.com/slack)!
@@ -150,4 +147,4 @@ RisingWave is distributed under the Apache License (Version 2.0). Please refer t
 
 ## Contributing
 
-Thanks for your interest in contributing to the project! Please refer to [contribution guidelines](CONTRIBUTING.md) for more information.
+Thanks for your interest in contributing to the project! Please refer to [RisingWave Developer Guide](https://risingwavelabs.github.io/risingwave/) for more information.
diff --git a/ci/Dockerfile b/ci/Dockerfile
index 569b3f9a4b675..b14454859a791 100644
--- a/ci/Dockerfile
+++ b/ci/Dockerfile
@@ -68,16 +68,16 @@ ENV CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse
 
 # install build tools
 RUN curl -L --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/cargo-bins/cargo-binstall/main/install-from-binstall-release.sh | bash
-RUN cargo binstall -y --no-symlinks cargo-llvm-cov cargo-nextest cargo-hakari cargo-sort cargo-cache cargo-audit \
+RUN cargo binstall -y --locked --no-symlinks cargo-llvm-cov cargo-nextest cargo-hakari cargo-sort cargo-cache cargo-audit \
     cargo-make@0.37.9 \
     sqllogictest-bin@0.21.0 \
     sccache@0.7.4 \
     && cargo cache -a \
     && rm -rf "/root/.cargo/registry/index" \
     && rm -rf "/root/.cargo/registry/cache" \
-    && rm -rf "/root/.cargo/git/db"
+    && rm -rf "/root/.cargo/git/db" \
+    && cargo uninstall cargo-cache
 RUN cargo install cargo-dylint@3.1.0 dylint-link@3.1.0
-RUN cargo uninstall cargo-cache
 
 # install risedev
 COPY <<EOF /root/.cargo/bin/risedev
diff --git a/ci/build-ci-image.sh b/ci/build-ci-image.sh
index 58f0c74d22922..6602509824e05 100755
--- a/ci/build-ci-image.sh
+++ b/ci/build-ci-image.sh
@@ -10,7 +10,7 @@ cat ../rust-toolchain
 # shellcheck disable=SC2155
 
 # REMEMBER TO ALSO UPDATE ci/docker-compose.yml
-export BUILD_ENV_VERSION=v20240628
+export BUILD_ENV_VERSION=v20240731
 
 export BUILD_TAG="public.ecr.aws/w1p7b4n3/rw-build-env:${BUILD_ENV_VERSION}"
 
diff --git a/ci/docker-compose.yml b/ci/docker-compose.yml
index 66b3ea3eb4bc6..78ad69c0995ab 100644
--- a/ci/docker-compose.yml
+++ b/ci/docker-compose.yml
@@ -12,7 +12,7 @@ services:
       interval: 5s
       timeout: 5s
       retries: 5
-    command: [ "postgres", "-c", "wal_level=logical" ]
+    command: [ "postgres", "-c", "wal_level=logical", "-c", "max_replication_slots=30" ]
 
   mysql:
     image: mysql:8.0
@@ -71,9 +71,10 @@ services:
       retries: 5
 
   source-test-env:
-    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20240628
+    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20240731
     depends_on:
       - mysql
+      - sqlserver-server
       - db
       - message_queue
       - schemaregistry
@@ -84,7 +85,7 @@ services:
       - ..:/risingwave
 
   sink-test-env:
-    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20240628
+    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20240731
     depends_on:
       - mysql
       - db
@@ -107,12 +108,12 @@ services:
 
 
   rw-build-env:
-    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20240628
+    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20240731
     volumes:
       - ..:/risingwave
 
   ci-flamegraph-env:
-    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20240628
+    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20240731
     # NOTE(kwannoel): This is used in order to permit
     # syscalls for `nperf` (perf_event_open),
     # so it can do CPU profiling.
@@ -123,7 +124,7 @@ services:
       - ..:/risingwave
 
   regress-test-env:
-    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20240628
+    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20240731
     depends_on:
       db:
         condition: service_healthy
@@ -215,6 +216,7 @@ services:
     environment:
       ACCEPT_EULA: 'Y'
       SA_PASSWORD: 'SomeTestOnly@SA'
+      MSSQL_AGENT_ENABLED: "true"
 
   starrocks-fe-server:
     container_name: starrocks-fe-server
diff --git a/ci/rust-toolchain b/ci/rust-toolchain
index 92ef4a5ff27be..6bc57a2a65d8f 100644
--- a/ci/rust-toolchain
+++ b/ci/rust-toolchain
@@ -4,4 +4,4 @@
 # 3. (optional) **follow the instructions in lints/README.md** to update the toolchain and dependencies for lints
 
 [toolchain]
-channel = "nightly-2024-03-12"
+channel = "nightly-2024-06-06"
diff --git a/ci/scripts/build-other.sh b/ci/scripts/build-other.sh
index 65c50462f97a0..d59e8ea876ccf 100755
--- a/ci/scripts/build-other.sh
+++ b/ci/scripts/build-other.sh
@@ -5,9 +5,9 @@ set -euo pipefail
 
 source ci/scripts/common.sh
 
-
 echo "--- Build Rust UDF"
 cd e2e_test/udf/wasm
+rustup target add wasm32-wasi
 cargo build --release
 cd ../../..
 
diff --git a/ci/scripts/common.sh b/ci/scripts/common.sh
index 3b31afeef253d..ac64d1a7a89cc 100755
--- a/ci/scripts/common.sh
+++ b/ci/scripts/common.sh
@@ -15,6 +15,8 @@ export MCLI_DOWNLOAD_BIN=https://rw-ci-deps-dist.s3.amazonaws.com/mc
 export GCLOUD_DOWNLOAD_TGZ=https://rw-ci-deps-dist.s3.amazonaws.com/google-cloud-cli-475.0.0-linux-x86_64.tar.gz
 export NEXTEST_HIDE_PROGRESS_BAR=true
 export RW_TELEMETRY_TYPE=test
+export RW_SECRET_STORE_PRIVATE_KEY_HEX="0123456789abcdef"
+
 unset LANG
 if [ -n "${BUILDKITE_COMMIT:-}" ]; then
   export GIT_SHA=$BUILDKITE_COMMIT
diff --git a/ci/scripts/deterministic-recovery-test.sh b/ci/scripts/deterministic-recovery-test.sh
index 4dd2c1ec88936..1a400d4ade9e0 100755
--- a/ci/scripts/deterministic-recovery-test.sh
+++ b/ci/scripts/deterministic-recovery-test.sh
@@ -13,7 +13,9 @@ export RUST_LOG="risingwave_meta::barrier::recovery=debug,\
 risingwave_meta::manager::catalog=debug,\
 risingwave_meta::rpc::ddl_controller=debug,\
 risingwave_meta::barrier::mod=debug,\
-risingwave_simulation=debug"
+risingwave_simulation=debug,\
+risingwave_meta::stream::stream_manager=debug,\
+risingwave_meta::barrier::progress=debug"
 
 # Extra logs you can enable if the existing trace does not give enough info.
 #risingwave_stream::executor::backfill=trace,
diff --git a/ci/scripts/docker-hdfs.sh b/ci/scripts/docker-hdfs.sh
index 5b46c3be50397..27bb48bbf83d8 100755
--- a/ci/scripts/docker-hdfs.sh
+++ b/ci/scripts/docker-hdfs.sh
@@ -10,10 +10,10 @@ BUILDKITE_COMMIT="HDFS_$(echo $RANDOM | md5sum | head -c 20;)"
 
 java_home_path=$(uname -m)
 if [ "$arch" = "arm64" ] || [ "$arch" = "aarch64" ]; then
-    java_home_path="/usr/lib/jvm/java-11-openjdk-arm64"
+    java_home_path="/usr/lib/jvm/java-17-openjdk-arm64"
 else
 # x86_64
-    java_home_path="/usr/lib/jvm/java-11-openjdk-amd64"
+    java_home_path="/usr/lib/jvm/java-17-openjdk-amd64"
 fi
 echo $java_home_path
 
diff --git a/ci/scripts/e2e-source-test.sh b/ci/scripts/e2e-source-test.sh
index c4b4713af81cc..b6eff317a79ed 100755
--- a/ci/scripts/e2e-source-test.sh
+++ b/ci/scripts/e2e-source-test.sh
@@ -63,6 +63,14 @@ echo "--- starting risingwave cluster"
 RUST_LOG="debug,risingwave_stream=info,risingwave_batch=info,risingwave_storage=info" \
 risedev ci-start ci-1cn-1fe-with-recovery
 
+echo "--- Install sql server client"
+curl https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add -
+curl https://packages.microsoft.com/config/ubuntu/20.04/prod.list | sudo tee /etc/apt/sources.list.d/msprod.list
+apt-get update -y
+ACCEPT_EULA=Y DEBIAN_FRONTEND=noninteractive apt-get install -y mssql-tools unixodbc-dev
+export PATH="/opt/mssql-tools/bin/:$PATH"
+sleep 2
+
 echo "--- mongodb cdc test"
 # install the mongo shell
 wget http://archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2_amd64.deb
@@ -79,6 +87,7 @@ risedev slt './e2e_test/source/cdc/mongodb/**/*.slt'
 
 echo "--- inline cdc test"
 export MYSQL_HOST=mysql MYSQL_TCP_PORT=3306 MYSQL_PWD=123456
+export SQLCMDSERVER=sqlserver-server SQLCMDUSER=SA SQLCMDPASSWORD="SomeTestOnly@SA" SQLCMDDBNAME=mydb SQLCMDPORT=1433
 risedev slt './e2e_test/source/cdc_inline/**/*.slt'
 
 echo "--- opendal source test"
diff --git a/ci/scripts/e2e-sqlserver-sink-test.sh b/ci/scripts/e2e-sqlserver-sink-test.sh
index f1f62941375ce..30032c66a6286 100755
--- a/ci/scripts/e2e-sqlserver-sink-test.sh
+++ b/ci/scripts/e2e-sqlserver-sink-test.sh
@@ -74,6 +74,7 @@ if [[ ${#actual[@]} -eq ${#expected[@]} && ${actual[@]} == ${expected[@]} ]]; th
 else
   cat ./query_result.txt
   echo "The output is not as expected."
+  exit 1
 fi
 
 echo "--- Kill cluster"
diff --git a/ci/scripts/e2e-time-travel-test.sh b/ci/scripts/e2e-time-travel-test.sh
new file mode 100755
index 0000000000000..d69556b3c2d75
--- /dev/null
+++ b/ci/scripts/e2e-time-travel-test.sh
@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+
+# Exits as soon as any line fails.
+set -euo pipefail
+
+source ci/scripts/common.sh
+
+while getopts 'p:' opt; do
+    case ${opt} in
+        p )
+            profile=$OPTARG
+            ;;
+        \? )
+            echo "Invalid Option: -$OPTARG" 1>&2
+            exit 1
+            ;;
+        : )
+            echo "Invalid option: $OPTARG requires an argument" 1>&2
+            ;;
+    esac
+done
+shift $((OPTIND -1))
+
+sudo apt install sqlite3 -y
+download_and_prepare_rw "$profile" common
+
+echo "--- starting risingwave cluster"
+risedev ci-start ci-time-travel
+sleep 1
+
+sqllogictest -p 4566 -d dev './e2e_test/time_travel/*.slt'
+
+echo "--- verify time travel metadata"
+sleep 30 # ensure another time travel version snapshot has been taken
+version_snapshot_count=$(sqlite3 .risingwave/data/sqlite/metadata.db "select count(*) from hummock_time_travel_version;")
+if [ "$version_snapshot_count" -le 1 ]; then
+  echo "test failed: too few version_snapshot_count, actual ${version_snapshot_count}"
+  exit 1
+fi
+
+echo "--- Kill cluster"
+risedev ci-kill
+
diff --git a/ci/scripts/find-regression.py b/ci/scripts/find-regression.py
new file mode 100755
index 0000000000000..e5106c9d892e0
--- /dev/null
+++ b/ci/scripts/find-regression.py
@@ -0,0 +1,262 @@
+#!/usr/bin/env python3
+
+import subprocess
+import unittest
+import os
+import sys
+
+'''
+@kwannoel
+This script is used to find the commit that introduced a regression in the codebase.
+It uses binary search to find the regressed commit.
+It works as follows:
+1. Use the start (inclusive) and end (exclusive) bounds, find the middle commit.
+   e.g. given commit 0->1(start)->2->3->4(bad), start will be 1, end will be 4. Then the middle commit is (1+4)//2 = 2
+        given commit 0->1(start)->2->3(bad)->4, start will be 1, end will be 3. Then the middle commit is (1+3)//2 = 2
+        given commit 0->1(start)->2(bad), start will be 1, end will be 2. Then the middle commit is (1+2)//2 = 1.
+        given commit 0->1(start,bad), start will be 1, end will be 1. We just return the bad commit (1) immediately.
+2. Run the pipeline on the middle commit.
+3. If the pipeline fails, the regression is in the first half of the commits. Recurse (start, mid)
+4. If the pipeline passes, the regression is in the second half of the commits. Recurse (mid+1, end)
+5. If start>=end, return start as the regressed commit.
+
+We won't run the entire pipeline, only steps specified by the CI_STEPS environment variable.
+
+For step (2), we need to check its outcome and only run the next step, if the outcome is successful.
+'''
+
+
+def format_step(env):
+    commit = get_bisect_commit(env["GOOD_COMMIT"], env["BAD_COMMIT"])
+    step = f'''
+cat <<- YAML | buildkite-agent pipeline upload
+steps:
+  - label: "run-{commit}"
+    key: "run-{commit}"
+    trigger: "main-cron"
+    soft_fail: true
+    build:
+      branch: {env["BISECT_BRANCH"]}
+      commit: {commit}
+      env:
+        CI_STEPS: {env['CI_STEPS']}
+  - wait
+  - label: 'check'
+    command: |
+        GOOD_COMMIT={env['GOOD_COMMIT']} BAD_COMMIT={env['BAD_COMMIT']} BISECT_BRANCH={env['BISECT_BRANCH']} CI_STEPS=\'{env['CI_STEPS']}\' ci/scripts/find-regression.py check
+YAML'''
+    return step
+
+
+def report_step(commit):
+    step = f'''
+cat <<- YAML | buildkite-agent pipeline upload
+steps:
+  - label: "Regressed Commit: {commit}"
+    command: "echo 'Regressed Commit: {commit}'"
+YAML'''
+    print(f"--- reporting regression commit: {commit}")
+    result = subprocess.run(step, shell=True)
+    if result.returncode != 0:
+        print(f"stderr: {result.stderr}")
+        print(f"stdout: {result.stdout}")
+        sys.exit(1)
+
+
+# Triggers a buildkite job to run the pipeline on the given commit, with the specified tests.
+def run_pipeline(env):
+    step = format_step(env)
+    print(f"--- running upload pipeline for step\n{step}")
+    result = subprocess.run(step, shell=True)
+    if result.returncode != 0:
+        print(f"stderr: {result.stderr}")
+        print(f"stdout: {result.stdout}")
+        sys.exit(1)
+
+
+# Number of commits for [start, end)
+def get_number_of_commits(start, end):
+    cmd = f"git rev-list --count {start}..{end}"
+    result = subprocess.run([cmd], shell=True, capture_output=True, text=True)
+    if result.returncode != 0:
+        print(f"stderr: {result.stderr}")
+        print(f"stdout: {result.stdout}")
+        sys.exit(1)
+    return int(result.stdout)
+
+
+def get_bisect_commit(start, end):
+    number_of_commits = get_number_of_commits(start, end)
+    commit_offset = number_of_commits // 2
+    if commit_offset == 0:
+        return start
+
+    cmd = f"git rev-list --reverse {start}..{end} | head -n {commit_offset} | tail -n 1"
+    result = subprocess.run([cmd], shell=True, capture_output=True, text=True)
+    if result.returncode != 0:
+        print(f"stderr: {result.stderr}")
+        print(f"stdout: {result.stdout}")
+        sys.exit(1)
+    return result.stdout.strip()
+
+
+def get_commit_after(branch, commit):
+    cmd = f"git log --reverse --ancestry-path {commit}..origin/{branch} --format=%H | head -n 1"
+    result = subprocess.run([cmd], shell=True, capture_output=True, text=True)
+
+    if result.returncode != 0:
+        print(f"stderr: {result.stderr}")
+        print(f"stdout: {result.stdout}")
+        sys.exit(1)
+
+    return result.stdout.strip()
+
+
+def get_env():
+    env = {
+        "GOOD_COMMIT": os.environ['GOOD_COMMIT'],
+        "BAD_COMMIT": os.environ['BAD_COMMIT'],
+        "BISECT_BRANCH": os.environ['BISECT_BRANCH'],
+        "CI_STEPS": os.environ['CI_STEPS'],
+    }
+
+    print(f'''
+GOOD_COMMIT={env["GOOD_COMMIT"]}
+BAD_COMMIT={env["BAD_COMMIT"]}
+BISECT_BRANCH={env["BISECT_BRANCH"]}
+CI_STEPS={env["CI_STEPS"]}
+        ''')
+
+    return env
+
+
+def fetch_branch_commits(branch):
+    cmd = f"git fetch -q origin {branch}"
+    result = subprocess.run([cmd], shell=True)
+    if result.returncode != 0:
+        print(f"stderr: {result.stderr}")
+        print(f"stdout: {result.stdout}")
+        sys.exit(1)
+
+
+def main():
+    cmd = sys.argv[1]
+
+    if cmd == "start":
+        print("--- start bisecting")
+        env = get_env()
+        fetch_branch_commits(env["BISECT_BRANCH"])
+        run_pipeline(env)
+    elif cmd == "check":
+        print("--- check pipeline outcome")
+        env = get_env()
+        fetch_branch_commits(env["BISECT_BRANCH"])
+        commit = get_bisect_commit(env["GOOD_COMMIT"], env["BAD_COMMIT"])
+        step = f"run-{commit}"
+        cmd = f"buildkite-agent step get outcome --step {step}"
+        outcome = subprocess.run(cmd, shell=True, capture_output=True, text=True)
+
+        if outcome.returncode != 0:
+            print(f"stderr: {outcome.stderr}")
+            print(f"stdout: {outcome.stdout}")
+            sys.exit(1)
+
+        outcome = outcome.stdout.strip()
+        if outcome == "soft_failed":
+            print(f"commit failed: {commit}")
+            env["BAD_COMMIT"] = commit
+        elif outcome == "passed":
+            print(f"commit passed: {commit}")
+            env["GOOD_COMMIT"] = get_commit_after(env["BISECT_BRANCH"], commit)
+        else:
+            print(f"invalid outcome: {outcome}")
+            sys.exit(1)
+
+        if env["GOOD_COMMIT"] == env["BAD_COMMIT"]:
+            report_step(env["GOOD_COMMIT"])
+            return
+        else:
+            print(f"run next iteration, start: {env['GOOD_COMMIT']}, end: {env['BAD_COMMIT']}")
+            run_pipeline(env)
+    else:
+        print(f"invalid cmd: {cmd}")
+        sys.exit(1)
+
+
+# For the tests, we use RisingWave's sequence of commits, from earliest to latest:
+# 617d23ddcac88ced87b96a2454c9217da0fe7915
+# 72f70960226680e841a8fbdd09c79d74609f27a2
+# 5c7b556ea60d136c5bccf1b1f7e313d2f9c79ef0
+# 9ca415a9998a5e04e021c899fb66d93a17931d4f
+class Test(unittest.TestCase):
+    def test_get_commit_after(self):
+        fetch_branch_commits("kwannoel/find-regress")
+        commit = get_commit_after("kwannoel/find-regress", "72f70960226680e841a8fbdd09c79d74609f27a2")
+        self.assertEqual(commit, "5c7b556ea60d136c5bccf1b1f7e313d2f9c79ef0")
+        commit2 = get_commit_after("kwannoel/find-regress", "617d23ddcac88ced87b96a2454c9217da0fe7915")
+        self.assertEqual(commit2, "72f70960226680e841a8fbdd09c79d74609f27a2")
+        commit3 = get_commit_after("kwannoel/find-regress", "5c7b556ea60d136c5bccf1b1f7e313d2f9c79ef0")
+        self.assertEqual(commit3, "9ca415a9998a5e04e021c899fb66d93a17931d4f")
+
+    def test_get_number_of_commits(self):
+        fetch_branch_commits("kwannoel/find-regress")
+        n = get_number_of_commits("72f70960226680e841a8fbdd09c79d74609f27a2",
+                                  "9ca415a9998a5e04e021c899fb66d93a17931d4f")
+        self.assertEqual(n, 2)
+        n2 = get_number_of_commits("617d23ddcac88ced87b96a2454c9217da0fe7915",
+                                   "9ca415a9998a5e04e021c899fb66d93a17931d4f")
+        self.assertEqual(n2, 3)
+        n3 = get_number_of_commits("72f70960226680e841a8fbdd09c79d74609f27a2",
+                                   "5c7b556ea60d136c5bccf1b1f7e313d2f9c79ef0")
+        self.assertEqual(n3, 1)
+
+    def test_get_bisect_commit(self):
+        fetch_branch_commits("kwannoel/find-regress")
+        commit = get_bisect_commit("72f70960226680e841a8fbdd09c79d74609f27a2",
+                                   "9ca415a9998a5e04e021c899fb66d93a17931d4f")
+        self.assertEqual(commit, "5c7b556ea60d136c5bccf1b1f7e313d2f9c79ef0")
+        commit2 = get_bisect_commit("617d23ddcac88ced87b96a2454c9217da0fe7915",
+                                    "9ca415a9998a5e04e021c899fb66d93a17931d4f")
+        self.assertEqual(commit2, "72f70960226680e841a8fbdd09c79d74609f27a2")
+        commit3 = get_bisect_commit("72f70960226680e841a8fbdd09c79d74609f27a2",
+                                    "5c7b556ea60d136c5bccf1b1f7e313d2f9c79ef0")
+        self.assertEqual(commit3, "72f70960226680e841a8fbdd09c79d74609f27a2")
+
+    def test_format_step(self):
+        fetch_branch_commits("kwannoel/find-regress")
+        self.maxDiff = None
+        env = {
+            "GOOD_COMMIT": "72f70960226680e841a8fbdd09c79d74609f27a2",
+            "BAD_COMMIT": "9ca415a9998a5e04e021c899fb66d93a17931d4f",
+            "BISECT_BRANCH": "kwannoel/find-regress",
+            "CI_STEPS": "test"
+        }
+        step = format_step(env)
+        self.assertEqual(
+            step,
+            '''
+cat <<- YAML | buildkite-agent pipeline upload
+steps:
+  - label: "run-5c7b556ea60d136c5bccf1b1f7e313d2f9c79ef0"
+    key: "run-5c7b556ea60d136c5bccf1b1f7e313d2f9c79ef0"
+    trigger: "main-cron"
+    soft_fail: true
+    build:
+      branch: kwannoel/find-regress
+      commit: 5c7b556ea60d136c5bccf1b1f7e313d2f9c79ef0
+      env:
+        CI_STEPS: test
+  - wait
+  - label: 'check'
+    command: |
+        GOOD_COMMIT=72f70960226680e841a8fbdd09c79d74609f27a2 BAD_COMMIT=9ca415a9998a5e04e021c899fb66d93a17931d4f BISECT_BRANCH=kwannoel/find-regress CI_STEPS='test' ci/scripts/find-regression.py check
+YAML'''
+        )
+
+
+if __name__ == "__main__":
+    # You can run tests by just doing ./ci/scripts/find-regression.py
+    if len(sys.argv) == 1:
+        unittest.main()
+    else:
+        main()
diff --git a/ci/scripts/release.sh b/ci/scripts/release.sh
index 94fd38e2c9c75..2235d8da85ead 100755
--- a/ci/scripts/release.sh
+++ b/ci/scripts/release.sh
@@ -71,9 +71,9 @@ if [ "${ARCH}" == "aarch64" ]; then
   # see https://github.com/tikv/jemallocator/blob/802969384ae0c581255f3375ee2ba774c8d2a754/jemalloc-sys/build.rs#L218
   export JEMALLOC_SYS_WITH_LG_PAGE=16
 fi
-OPENSSL_STATIC=1 cargo build -p risingwave_cmd_all --features "rw-static-link" --features external-udf --features wasm-udf --features js-udf --profile release
-OPENSSL_STATIC=1 cargo build -p risingwave_cmd --bin risectl --features "rw-static-link" --profile release
-cd target/release && chmod +x risingwave risectl
+OPENSSL_STATIC=1 cargo build -p risingwave_cmd_all --features "rw-static-link" --features external-udf --features wasm-udf --features js-udf --profile production
+OPENSSL_STATIC=1 cargo build -p risingwave_cmd --bin risectl --features "rw-static-link" --profile production
+cd target/production && chmod +x risingwave risectl
 
 echo "--- Upload nightly binary to s3"
 if [ "${BUILDKITE_SOURCE}" == "schedule" ]; then
@@ -90,7 +90,7 @@ cd "${REPO_ROOT}"/java && mvn -B package -Dmaven.test.skip=true -Dno-build-rust
 if [[ -n "${BUILDKITE_TAG}" ]]; then
   echo "--- Collect all release assets"
   cd "${REPO_ROOT}" && mkdir release-assets && cd release-assets
-  cp -r "${REPO_ROOT}"/target/release/* .
+  cp -r "${REPO_ROOT}"/target/production/* .
   mv "${REPO_ROOT}"/java/connector-node/assembly/target/risingwave-connector-1.0.0.tar.gz risingwave-connector-"${BUILDKITE_TAG}".tar.gz
   tar -zxvf risingwave-connector-"${BUILDKITE_TAG}".tar.gz libs
   ls -l
diff --git a/ci/scripts/run-e2e-test.sh b/ci/scripts/run-e2e-test.sh
index 4736f4aa53a8a..b4c00cec53fe8 100755
--- a/ci/scripts/run-e2e-test.sh
+++ b/ci/scripts/run-e2e-test.sh
@@ -27,6 +27,7 @@ if [[ $mode == "standalone" ]]; then
 fi
 
 if [[ $mode == "single-node" ]]; then
+  export RUST_MIN_STACK=4194304
   source ci/scripts/single-node-utils.sh
 fi
 
@@ -88,7 +89,7 @@ cluster_stop
 echo "--- e2e, $mode, batch"
 RUST_LOG="info,risingwave_stream=info,risingwave_batch=info,risingwave_storage=info" \
 cluster_start
-sqllogictest -p 4566 -d dev './e2e_test/ddl/**/*.slt' --junit "batch-ddl-${profile}"
+sqllogictest -p 4566 -d dev './e2e_test/ddl/**/*.slt' --junit "batch-ddl-${profile}" --label "can-use-recover"
 if [[ "$mode" != "single-node" ]]; then
   sqllogictest -p 4566 -d dev './e2e_test/background_ddl/basic.slt' --junit "batch-ddl-${profile}"
 fi
@@ -133,7 +134,6 @@ sqllogictest -p 4566 -d dev './e2e_test/udf/wasm_udf.slt'
 sqllogictest -p 4566 -d dev './e2e_test/udf/rust_udf.slt'
 sqllogictest -p 4566 -d dev './e2e_test/udf/js_udf.slt'
 sqllogictest -p 4566 -d dev './e2e_test/udf/python_udf.slt'
-sqllogictest -p 4566 -d dev './e2e_test/udf/deno_udf.slt'
 
 echo "--- Kill cluster"
 cluster_stop
diff --git a/ci/scripts/s3-source-test.sh b/ci/scripts/s3-source-test.sh
index 532223693a215..4ae2397194762 100755
--- a/ci/scripts/s3-source-test.sh
+++ b/ci/scripts/s3-source-test.sh
@@ -32,7 +32,7 @@ echo "--- starting risingwave cluster with connector node"
 risedev ci-start ci-1cn-1fe
 
 echo "--- Run test"
-python3 -m pip install --break-system-packages minio psycopg2-binary opendal
+python3 -m pip install --break-system-packages minio psycopg2-binary opendal pandas
 if [[ -v format_type ]]; then
   python3 e2e_test/s3/"$script" "$format_type"
 else
diff --git a/ci/scripts/single-node-utils.sh b/ci/scripts/single-node-utils.sh
index 8f7240d3d906a..f882084197af8 100755
--- a/ci/scripts/single-node-utils.sh
+++ b/ci/scripts/single-node-utils.sh
@@ -5,6 +5,8 @@ set -euo pipefail
 export RW_PREFIX=$PWD/.risingwave
 export PREFIX_BIN=./target/debug
 export PREFIX_LOG=$RW_PREFIX/log
+export PREFIX_CONFIG=$RW_PREFIX/config
+export RISEDEV=1 # as if we are running in RiseDev
 
 # You can fill up this section by consulting
 # .risingwave/log/risedev.log, after calling `risedev d full`.
@@ -12,6 +14,7 @@ export PREFIX_LOG=$RW_PREFIX/log
 start_single_node() {
   mkdir -p "$HOME/.risingwave/state_store"
   mkdir -p "$HOME/.risingwave/meta_store"
+  mkdir -p .risingwave/config
   RUST_BACKTRACE=1 "$PREFIX_BIN"/risingwave >"$1" 2>&1
 }
 
diff --git a/ci/workflows/main-cron-bisect.yml b/ci/workflows/main-cron-bisect.yml
new file mode 100644
index 0000000000000..ab8cebd234d74
--- /dev/null
+++ b/ci/workflows/main-cron-bisect.yml
@@ -0,0 +1,12 @@
+auto-retry: &auto-retry
+  automatic:
+    # Agent terminated because the AWS EC2 spot instance killed by AWS.
+    - signal_reason: agent_stop
+      limit: 3
+
+steps:
+  - label: "find regressed step"
+    key: "find-regressed-step"
+    command: "GOOD_COMMIT=$GOOD_COMMIT BAD_COMMIT=$BAD_COMMIT BISECT_BRANCH=$BISECT_BRANCH CI_STEPS=$CI_STEPS ci/scripts/find-regression.py start"
+    if: build.env("CI_STEPS") != null
+    retry: *auto-retry
diff --git a/ci/workflows/main-cron.yml b/ci/workflows/main-cron.yml
index 13c98c6bff9f8..17dc713818ff0 100644
--- a/ci/workflows/main-cron.yml
+++ b/ci/workflows/main-cron.yml
@@ -8,6 +8,8 @@ steps:
   - label: "build"
     command: "ci/scripts/build.sh -p ci-release"
     key: "build"
+    if: |
+      build.env("CI_STEPS") !~ /(^|,)disable-build(,|$$)/
     plugins:
       - docker-compose#v5.1.0:
           run: rw-build-env
@@ -19,6 +21,8 @@ steps:
   - label: "build other components"
     command: "ci/scripts/build-other.sh"
     key: "build-other"
+    if: |
+      build.env("CI_STEPS") !~ /(^|,)disable-build(,|$$)/
     plugins:
       - seek-oss/aws-sm#v2.3.1:
           env:
@@ -35,6 +39,8 @@ steps:
   - label: "build simulation test"
     command: "ci/scripts/build-simulation.sh"
     key: "build-simulation"
+    if: |
+      build.env("CI_STEPS") !~ /(^|,)disable-build(,|$$)/
     plugins:
       - docker-compose#v5.1.0:
           run: rw-build-env
@@ -46,6 +52,8 @@ steps:
   - label: "docslt"
     command: "ci/scripts/docslt.sh"
     key: "docslt"
+    if: |
+      build.env("CI_STEPS") !~ /(^|,)disable-build(,|$$)/
     plugins:
       - docker-compose#v5.1.0:
           run: rw-build-env
@@ -78,9 +86,9 @@ steps:
     key: "slow-e2e-test-release"
     command: "ci/scripts/slow-e2e-test.sh -p ci-release -m ci-3streaming-2serving-3fe"
     if: |
-        !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
-        || build.pull_request.labels includes "ci/run-slow-e2e-tests"
-        || build.env("CI_STEPS") =~ /(^|,)slow-e2e-tests?(,|$$)/
+      !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
+      || build.pull_request.labels includes "ci/run-slow-e2e-tests"
+      || build.env("CI_STEPS") =~ /(^|,)slow-e2e-tests?(,|$$)/
     depends_on:
       - "build"
       - "build-other"
@@ -478,6 +486,50 @@ steps:
     timeout_in_minutes: 25
     retry: *auto-retry
 
+  - label: "S3_v2 source new file check on AWS (json)"
+    key: "s3-v2-source-new-file-check-aws"
+    command: "ci/scripts/s3-source-test.sh -p ci-release -s fs_source_v2_new_file.py"
+    if: |
+      !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
+      || build.pull_request.labels includes "ci/run-s3-source-tests"
+      || build.env("CI_STEPS") =~ /(^|,)s3-source-tests?(,|$$)/
+    depends_on: build
+    plugins:
+      - seek-oss/aws-sm#v2.3.1:
+          env:
+            S3_SOURCE_TEST_CONF: ci_s3_source_test_aws
+      - docker-compose#v5.1.0:
+          run: rw-build-env
+          config: ci/docker-compose.yml
+          mount-buildkite-agent: true
+          environment:
+            - S3_SOURCE_TEST_CONF
+      - ./ci/plugins/upload-failure-logs
+    timeout_in_minutes: 25
+    retry: *auto-retry
+
+  - label: "S3_v2 source check on parquet file"
+    key: "s3-v2-source-check-parquet-file"
+    command: "ci/scripts/s3-source-test.sh -p ci-release -s fs_parquet_source.py"
+    if: |
+      !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
+      || build.pull_request.labels includes "ci/run-s3-source-tests"
+      || build.env("CI_STEPS") =~ /(^|,)s3-source-tests?(,|$$)/
+    depends_on: build
+    plugins:
+      - seek-oss/aws-sm#v2.3.1:
+          env:
+            S3_SOURCE_TEST_CONF: ci_s3_source_test_aws
+      - docker-compose#v5.1.0:
+          run: rw-build-env
+          config: ci/docker-compose.yml
+          mount-buildkite-agent: true
+          environment:
+            - S3_SOURCE_TEST_CONF
+      - ./ci/plugins/upload-failure-logs
+    timeout_in_minutes: 25
+    retry: *auto-retry
+
   - label: "S3_v2 source batch read on AWS (json parser)"
     key: "s3-v2-source-batch-read-check-aws-json-parser"
     command: "ci/scripts/s3-source-test.sh -p ci-release -s fs_source_batch.py -t json"
@@ -605,8 +657,7 @@ steps:
   - label: "upload micro-benchmark"
     key: "upload-micro-benchmarks"
     if: |
-      build.branch == "main"
-      || !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
+      !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
       || build.pull_request.labels includes "ci/run-micro-benchmarks"
       || build.env("CI_STEPS") =~ /(^|,)micro-benchmarks?(,|$$)/
     command:
@@ -692,7 +743,7 @@ steps:
           config: ci/docker-compose.yml
           mount-buildkite-agent: true
       - ./ci/plugins/upload-failure-logs
-    timeout_in_minutes: 22
+    timeout_in_minutes: 24
     retry: *auto-retry
 
   - label: "e2e standalone binary test"
@@ -830,13 +881,33 @@ steps:
     timeout_in_minutes: 10
     retry: *auto-retry
 
-  - label: "end-to-end cassandra sink test"
-    key: "e2e-cassandra-sink-tests"
-    command: "ci/scripts/e2e-cassandra-sink-test.sh -p ci-release"
+# FIXME(xxhZs): https://github.com/risingwavelabs/risingwave/issues/17855
+#  - label: "end-to-end cassandra sink test"
+#    key: "e2e-cassandra-sink-tests"
+#    command: "ci/scripts/e2e-cassandra-sink-test.sh -p ci-release"
+#    if: |
+#      !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
+#      || build.pull_request.labels includes "ci/run-e2e-cassandra-sink-tests"
+#      || build.env("CI_STEPS") =~ /(^|,)e2e-cassandra-sink-tests?(,|$$)/
+#    depends_on:
+#      - "build"
+#      - "build-other"
+#    plugins:
+#      - docker-compose#v5.1.0:
+#          run: sink-test-env
+#          config: ci/docker-compose.yml
+#          mount-buildkite-agent: true
+#      - ./ci/plugins/upload-failure-logs
+#    timeout_in_minutes: 10
+#    retry: *auto-retry
+
+  - label: "end-to-end clickhouse sink test"
+    key: "e2e-clickhouse-sink-tests"
+    command: "ci/scripts/e2e-clickhouse-sink-test.sh -p ci-release"
     if: |
       !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
-      || build.pull_request.labels includes "ci/run-e2e-cassandra-sink-tests"
-      || build.env("CI_STEPS") =~ /(^|,)e2e-cassandra-sink-tests?(,|$$)/
+      || build.pull_request.labels includes "ci/run-e2e-clickhouse-sink-tests"
+      || build.env("CI_STEPS") =~ /(^|,)e2e-clickhouse-sink-tests?(,|$$)/
     depends_on:
       - "build"
       - "build-other"
@@ -849,19 +920,20 @@ steps:
     timeout_in_minutes: 10
     retry: *auto-retry
 
-  - label: "end-to-end clickhouse sink test"
-    key: "e2e-clickhouse-sink-tests"
-    command: "ci/scripts/e2e-clickhouse-sink-test.sh -p ci-release"
+  - label: "end-to-end time travel test"
+    key: "e2e-time-travel-tests"
+    command: "ci/scripts/e2e-time-travel-test.sh -p ci-release"
     if: |
       !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
-      || build.pull_request.labels includes "ci/run-e2e-clickhouse-sink-tests"
-      || build.env("CI_STEPS") =~ /(^|,)e2e-clickhouse-sink-tests?(,|$$)/
+      || build.pull_request.labels includes "ci/run-e2e-time-travel-tests"
+      || build.env("CI_STEPS") =~ /(^|,)e2e-time-travel-tests?(,|$$)/
     depends_on:
       - "build"
       - "build-other"
+      - "docslt"
     plugins:
       - docker-compose#v5.1.0:
-          run: sink-test-env
+          run: rw-build-env
           config: ci/docker-compose.yml
           mount-buildkite-agent: true
       - ./ci/plugins/upload-failure-logs
@@ -929,7 +1001,7 @@ steps:
     key: "e2e-mongodb-sink-tests"
     command: "ci/scripts/e2e-mongodb-sink-test.sh -p ci-release"
     if: |
-      !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
+      !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
       || build.pull_request.labels includes "ci/run-e2e-mongodb-sink-tests"
       || build.env("CI_STEPS") =~ /(^|,)e2e-mongodb-sink-tests?(,|$$)/
     depends_on:
@@ -1055,13 +1127,13 @@ steps:
 
   # Notification test.
   - key: "test-notify"
-    if: build.pull_request.labels includes "ci/main-cron/test-notify"
+    if: build.pull_request.labels includes "ci/main-cron/test-notify" || build.env("CI_STEPS") =~ /(^|,)test_notify(,|$$)/
     command: |
       bash -c 'echo test && exit -1'
 
   # Notification test.
   - key: "test-notify-2"
-    if: build.pull_request.labels includes "ci/main-cron/test-notify"
+    if: build.pull_request.labels includes "ci/main-cron/test-notify" || build.env("CI_STEPS") =~ /(^|,)test_notify(,|$$)/
     command: |
       bash -c 'echo test && exit -1'
 
@@ -1074,4 +1146,4 @@ steps:
   # This should be the LAST part of the main-cron file.
   - label: "trigger failed test notification"
     if: build.pull_request.labels includes "ci/main-cron/test-notify" || build.branch == "main"
-    command: "ci/scripts/notify.py"
+    command: "ci/scripts/notify.py"
\ No newline at end of file
diff --git a/ci/workflows/pull-request.yml b/ci/workflows/pull-request.yml
index ae8db23bbeb81..fd3e44f131c3f 100644
--- a/ci/workflows/pull-request.yml
+++ b/ci/workflows/pull-request.yml
@@ -333,6 +333,23 @@ steps:
     timeout_in_minutes: 10
     retry: *auto-retry
 
+  - label: "end-to-end time travel test"
+    key: "e2e-time-travel-tests"
+    command: "ci/scripts/e2e-time-travel-test.sh -p ci-dev"
+    if: build.pull_request.labels includes "ci/run-e2e-time-travel-tests" || build.env("CI_STEPS") =~ /(^|,)e2e-time-travel-tests?(,|$$)/
+    depends_on:
+      - "build"
+      - "build-other"
+      - "docslt"
+    plugins:
+      - docker-compose#v5.1.0:
+          run: rw-build-env
+          config: ci/docker-compose.yml
+          mount-buildkite-agent: true
+      - ./ci/plugins/upload-failure-logs
+    timeout_in_minutes: 15
+    retry: *auto-retry
+
   - label: "end-to-end sqlserver sink test"
     if: build.pull_request.labels includes "ci/run-e2e-sqlserver-sink-tests" || build.env("CI_STEPS") =~ /(^|,)e2e-sqlserver-sink-tests?(,|$$)/
     command: "ci/scripts/e2e-sqlserver-sink-test.sh -p ci-dev"
diff --git a/clippy.toml b/clippy.toml
index 551de0eb6c479..21b972376b0ed 100644
--- a/clippy.toml
+++ b/clippy.toml
@@ -39,3 +39,6 @@ doc-valid-idents = [
 avoid-breaking-exported-api = false
 upper-case-acronyms-aggressive = true
 too-many-arguments-threshold = 10
+ignore-interior-mutability = [
+    "risingwave_frontend::expr::ExprImpl" # XXX: Where does ExprImpl have interior mutability?
+]
diff --git a/docker/Dockerfile b/docker/Dockerfile
index 4cd5c5157dc42..dafee4cdc68d9 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -55,7 +55,7 @@ RUN cargo install flamegraph
 # COPY --from=rust-planner /risingwave/recipe.json recipe.json
 
 # # Build dependencies - this can be cached if the dependencies don't change
-# RUN cargo chef cook --release --recipe-path recipe.json
+# RUN cargo chef cook --profile production --recipe-path recipe.json
 
 FROM rust-base AS rust-builder
 
@@ -70,11 +70,11 @@ ENV ENABLE_BUILD_DASHBOARD=1
 ENV OPENSSL_STATIC=1
 
 RUN cargo fetch && \
-  cargo build -p risingwave_cmd_all --release --features "rw-static-link" --features all-udf && \
+  cargo build -p risingwave_cmd_all --profile production --features "rw-static-link" --features all-udf && \
   mkdir -p /risingwave/bin && \
-    mv /risingwave/target/release/risingwave /risingwave/bin/ && \
-    mv /risingwave/target/release/risingwave.dwp /risingwave/bin/ && \
-  cp ./target/release/build/tikv-jemalloc-sys-*/out/build/bin/jeprof /risingwave/bin/ && \
+    mv /risingwave/target/production/risingwave /risingwave/bin/ && \
+    mv /risingwave/target/production/risingwave.dwp /risingwave/bin/ && \
+  cp ./target/production/build/tikv-jemalloc-sys-*/out/build/bin/jeprof /risingwave/bin/ && \
   chmod +x /risingwave/bin/jeprof && \
   mkdir -p /risingwave/lib && cargo clean
 
diff --git a/docker/Dockerfile.hdfs b/docker/Dockerfile.hdfs
index b6eba07c421c0..ca967b41d0850 100644
--- a/docker/Dockerfile.hdfs
+++ b/docker/Dockerfile.hdfs
@@ -1,3 +1,5 @@
+# FIXME: this file is not well maintained compared to the main Dockerfile and may not work as expected.
+
 FROM ubuntu:24.04 AS base
 
 ENV LANG en_US.utf8
@@ -102,15 +104,15 @@ ENV OPENSSL_STATIC=1
 RUN cargo fetch && \
   cargo build -p risingwave_cmd_all --release -p risingwave_object_store --features hdfs-backend --features "rw-static-link" --features all-udf && \
   mkdir -p /risingwave/bin && \
-    mv /risingwave/target/release/risingwave /risingwave/bin/ && \
-    mv /risingwave/target/release/risingwave.dwp /risingwave/bin/ && \
+  mv /risingwave/target/release/risingwave /risingwave/bin/ && \
+  mv /risingwave/target/release/risingwave.dwp /risingwave/bin/ && \
   cp ./target/release/build/tikv-jemalloc-sys-*/out/build/bin/jeprof /risingwave/bin/ && \
   chmod +x /risingwave/bin/jeprof && \
   mkdir -p /risingwave/lib && cargo clean
 
 RUN cd /risingwave/java && mvn -B package -Dmaven.test.skip=true -Dno-build-rust && \
-    mkdir -p /risingwave/bin/connector-node && \
-    tar -zxvf /risingwave/java/connector-node/assembly/target/risingwave-connector-1.0.0.tar.gz -C /risingwave/bin/connector-node
+  mkdir -p /risingwave/bin/connector-node && \
+  tar -zxvf /risingwave/java/connector-node/assembly/target/risingwave-connector-1.0.0.tar.gz -C /risingwave/bin/connector-node
 
 FROM ubuntu:24.04 as image-base
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y install ca-certificates openjdk-17-jdk wget libsasl2-dev && rm -rf /var/lib/{apt,dpkg,cache,log}/
diff --git a/docker/aws/aws-build.sh b/docker/aws/aws-build.sh
index 40c01b44f792c..4742bea2bf178 100755
--- a/docker/aws/aws-build.sh
+++ b/docker/aws/aws-build.sh
@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+# FIXME: this script has not been maintained for a long time.
+
 set -e
 
 export DOCKER_BUILDKIT=1
diff --git a/docker/dashboards/risingwave-dev-dashboard.json b/docker/dashboards/risingwave-dev-dashboard.json
index 3e303bca36ae2..1a14442fc9fe0 100644
--- a/docker/dashboards/risingwave-dev-dashboard.json
+++ b/docker/dashboards/risingwave-dev-dashboard.json
@@ -1 +1 @@
-{"__inputs":[],"annotations":{"list":[]},"description":"RisingWave Dev Dashboard","editable":true,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"panels":[{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":1,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Information about actors","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":2,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true,"sortBy":[]},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"group(actor_info{job=~\"$job\",instance=~\"$node\"}) by (actor_id, fragment_id, compute_node)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"group(actor_info{job=~\"$job\",instance=~\"$node\"}) by (actor_id, fragment_id, compute_node)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true},"indexByName":{"actor_id":0,"compute_node":2,"fragment_id":1}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Information about state tables. Column `materialized_view_id` is the id of the materialized view that this state table belongs to.","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":3,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true,"sortBy":[]},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name, table_type, materialized_view_id, fragment_id, compaction_group_id)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name, table_type, materialized_view_id, fragment_id, compaction_group_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"State Table Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true},"indexByName":{"compaction_group_id":5,"fragment_id":4,"materialized_view_id":3,"table_id":0,"table_name":1,"table_type":2}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Actor count per compute node","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":4,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true,"sortBy":[]},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"count(actor_info{job=~\"$job\",instance=~\"$node\"}) by (compute_node)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"count(actor_info{job=~\"$job\",instance=~\"$node\"}) by (compute_node)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Count (Group By Compute Node)","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true},"indexByName":{"compaction_group_id":5,"fragment_id":4,"materialized_view_id":3,"table_id":0,"table_name":1,"table_type":2}}}],"transparent":false,"type":"table"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Actor/Table Id Info","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":1},"height":null,"hideTimeOverride":false,"id":5,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of each type of RisingWave components alive.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":6,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_type}}","metric":"","query":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The memory usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":7,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Memory","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Memory usage relative to k8s resource limit of container. Only works in K8s environment","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":8,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(avg by(namespace, pod) (container_memory_working_set_bytes{namespace=~\"$namespace\",pod=~\"$pod\",container=~\"$component\"})) / (  sum by(namespace, pod) (kube_pod_container_resource_limits{namespace=~\"$namespace\", pod=~\"$pod\", container=\"$component\", resource=\"memory\", unit=\"byte\"}))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg memory usage @ {{job}} @ {{instance}}","metric":"","query":"(avg by(namespace, pod) (container_memory_working_set_bytes{namespace=~\"$namespace\",pod=~\"$pod\",container=~\"$component\"})) / (  sum by(namespace, pod) (kube_pod_container_resource_limits{namespace=~\"$namespace\", pod=~\"$pod\", container=\"$component\", resource=\"memory\", unit=\"byte\"}))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Memory relative","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The CPU usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":9,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cpu usage (total) - {{job}} @ {{instance}}","metric":"","query":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (job, instance) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cpu usage (avg per core) - {{job}} @ {{instance}}","metric":"","query":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (job, instance) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"CPU usage relative to k8s resource limit of container. Only works in K8s environment","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":10,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\",container=~\"$component\",pod=~\"$pod\"}[$__rate_interval])) by (namespace, pod)) / (sum(kube_pod_container_resource_limits{namespace=~\"$namespace\",pod=~\"$pod\",container=~\"$component\", resource=\"cpu\"}) by (namespace, pod))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cpu usage @ {{job}} @ {{instance}}","metric":"","query":"(sum(rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\",container=~\"$component\",pod=~\"$pod\"}[$__rate_interval])) by (namespace, pod)) / (sum(kube_pod_container_resource_limits{namespace=~\"$namespace\",pod=~\"$pod\",container=~\"$component\", resource=\"cpu\"}) by (namespace, pod))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU relative","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"RW cluster can configure multiple meta nodes to achieve high availability. One is the leader and the rest are the followers.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":11,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(meta_num{job=~\"$job\",instance=~\"$node\"}) by (worker_addr,role)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_addr}} @ {{role}}","metric":"","query":"sum(meta_num{job=~\"$job\",instance=~\"$node\"}) by (worker_addr,role)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Meta Cluster","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Cluster Node","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":2},"height":null,"hideTimeOverride":false,"id":12,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The rate of successful recovery attempts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":13,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Recovery Successful Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of failed reocovery attempts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":14,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Failed recovery attempts","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Time spent in a successful recovery attempt","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":15,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency pmax - {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by (le) (rate(recovery_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by (le) (rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency avg","metric":"","query":"sum by (le) (rate(recovery_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by (le) (rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Recovery latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Recovery","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":3},"height":null,"hideTimeOverride":false,"id":16,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of barriers that have been ingested but not completely processed. This metric reflects the current level of congestion within the system.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":17,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all_barrier","metric":"","query":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"in_flight_barrier_nums{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"in_flight_barrier","metric":"","query":"in_flight_barrier_nums{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The time that the data between two consecutive barriers gets fully processed, i.e. the computation results are made durable into materialized views or sink to external systems. This metric shows to users the freshness of materialized views.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":18,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_avg","metric":"","query":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The duration from the last committed barrier's epoch time to the current time. This metric reflects the data freshness of the system. During this time, no new data has been committed.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":19,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"timestamp(last_committed_barrier_time{job=~\"$job\",instance=~\"$node\"}) - last_committed_barrier_time{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_pending_time","metric":"","query":"timestamp(last_committed_barrier_time{job=~\"$job\",instance=~\"$node\"}) - last_committed_barrier_time{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier pending time (secs)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":20,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","query":"sum(rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Each query is executed in parallel with a user-defined parallelism. This figure shows the throughput of each parallelism. The throughput of all the parallelism added up is equal to Source Throughput(rows).","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":21,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(source_partition_input_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor={{actor_id}} source={{source_id}} partition={{partition}} fragment_id={{fragment_id}}","metric":"","query":"rate(source_partition_input_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s) Per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of bytes read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":22,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum by (source_id, source_name, fragment_id)(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","query":"(sum by (source_id, source_name, fragment_id)(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Each query is executed in parallel with a user-defined parallelism. This figure shows the throughput of each parallelism. The throughput of all the parallelism added up is equal to Source Throughput(MB/s).","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":23,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor={{actor_id}} source={{source_id}} partition={{partition}} fragment_id={{fragment_id}}","metric":"","query":"(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))/(1000*1000)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s) Per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":24,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_source_backfill_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","query":"sum(rate(stream_source_backfill_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Backfill Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Monitor each source upstream, 0 means the upstream is not normal, 1 means the source is ready.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":25,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_status_is_up{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source_id={{source_id}}, source_name={{source_name}} @ {{instance}}","metric":"","query":"source_status_is_up{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Upstream Status","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Source Split Change Events frequency by source_id and actor_id","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":26,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_source_split_change_event_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_name}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_source_split_change_event_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Split Change Events frequency(events/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Kafka Consumer Lag Size by source_id, partition and actor_id","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":27,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_min(source_kafka_high_watermark{job=~\"$job\",instance=~\"$node\"} - on(source_id, partition) group_right() source_latest_message_id{job=~\"$job\",instance=~\"$node\"}, 0)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_id}} partition={{partition}} actor_id={{actor_id}}","metric":"","query":"clamp_min(source_kafka_high_watermark{job=~\"$job\",instance=~\"$node\"} - on(source_id, partition) group_right() source_latest_message_id{job=~\"$job\",instance=~\"$node\"}, 0)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kafka Consumer Lag Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of rows streamed into each sink per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":28,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id) * on(sink_id) group_left(sink_name) group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}}","metric":"","query":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id) * on(sink_id) group_left(sink_name) group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (sink_id, sink_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of rows streamed into each sink per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":29,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id, actor_id) * on(actor_id) group_left(sink_name) sink_info{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}} - actor {{actor_id}}","metric":"","query":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id, actor_id) * on(actor_id) group_left(sink_name) sink_info{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Throughput(rows/s) per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized view per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":30,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id) * on(table_id) group_left(table_name) group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mview {{table_id}} {{table_name}}","metric":"","query":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id) * on(table_id) group_left(table_name) group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized view per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":31,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) * on(fragment_id, table_id) group_left(table_name) table_info{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mview {{table_id}} {{table_name}} - actor {{actor_id}} fragment_id {{fragment_id}}","metric":"","query":"rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) * on(fragment_id, table_id) group_left(table_name) table_info{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s) per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been read from the backfill snapshot","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":32,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_snapshot_read_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_backfill_snapshot_read_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Snapshot Read Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been output from the backfill upstream","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":33,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_upstream_output_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_backfill_upstream_output_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Upstream Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The duration between the time point when the scheduled barrier needs to be sent and the time point when the barrier gets actually sent to all the compute nodes. Developers can thus detect any internal congestion.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":34,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_send_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_send_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_avg","metric":"","query":"rate(meta_barrier_send_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_send_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Send Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":35,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"max(sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_avg","metric":"","query":"max(sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier In-Flight Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":36,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p999 - {{instance}}","metric":"","query":"histogram_quantile(0.999, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_pmax - {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_avg - {{instance}}","metric":"","query":"sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Sync Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":37,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_wait_commit_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_wait_commit_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_avg","metric":"","query":"rate(meta_barrier_wait_commit_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_wait_commit_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Wait Commit Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of actors that have processed the earliest in-flight barriers per second. This metric helps users to detect potential congestion or stuck in the system.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":38,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_barrier_manager_progress{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"rate(stream_barrier_manager_progress{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Earliest In-Flight Barrier Progress","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":4},"height":null,"hideTimeOverride":false,"id":39,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been read from the cdc backfill snapshot","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":40,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_cdc_backfill_snapshot_read_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_cdc_backfill_snapshot_read_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CDC Backfill Snapshot Read Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been output from the cdc backfill upstream","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":41,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_cdc_backfill_upstream_output_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_cdc_backfill_upstream_output_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CDC Backfill Upstream Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":42,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lag p50 - {{table_name}}","metric":"","query":"histogram_quantile(0.5, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lag p99 - {{table_name}}","metric":"","query":"histogram_quantile(0.99, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lag pmax - {{table_name}}","metric":"","query":"histogram_quantile(1.0, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CDC Consume Lag Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":43,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(cdc_source_error{job=~\"$job\",instance=~\"$node\"}) by (connector_name, source_id, error_msg)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{connector_name}}: {{error_msg}} ({{source_id}})","metric":"","query":"sum(cdc_source_error{job=~\"$job\",instance=~\"$node\"}) by (connector_name, source_id, error_msg)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CDC Source Errors","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming CDC","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":5},"height":null,"hideTimeOverride":false,"id":44,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"We first record the total blocking duration(ns) of output buffer of each actor. It shows how much time it takes an actor to process a message, i.e. a barrier, a watermark or rows of data, on average. Then we divide this duration by 1 second and show it as a percentage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":45,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, downstream_fragment_id) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}->{{downstream_fragment_id}}","metric":"","query":"avg(rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, downstream_fragment_id) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Output Blocking Time Ratio (Backpressure)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":46,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_actor_input_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, upstream_fragment_id) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}<-{{upstream_fragment_id}}","metric":"","query":"avg(rate(stream_actor_input_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, upstream_fragment_id) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Input Blocking Time Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":47,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_actor_in_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, upstream_fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}<-{{upstream_fragment_id}}","metric":"","query":"sum(rate(stream_actor_in_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, upstream_fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_in_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}}","metric":"","query":"rate(stream_actor_in_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Input Throughput (rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":48,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_actor_out_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_actor_out_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_out_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}}","metric":"","query":"rate(stream_actor_out_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Output Throughput (rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The operator-level memory usage statistics collected by each LRU cache","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":49,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (table_id, desc)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table {{table_id}} desc: {{desc}}","metric":"","query":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (table_id, desc)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_memory_usage{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table {{table_id}} actor {{actor_id}} desc: {{desc}}","metric":"","query":"stream_memory_usage{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Memory Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Memory usage aggregated by materialized views","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":50,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id) group_left(materialized_view_id) table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized view {{materialized_view_id}}","metric":"","query":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id) group_left(materialized_view_id) table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Memory Usage of Materialized Views","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":51,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"temporal join cache miss, table_id {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"temporal join cache miss, table_id {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Temporal Join Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":52,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_materialize_cache_hit_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache hit count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_materialize_cache_hit_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_materialize_cache_total_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total cached count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_materialize_cache_total_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_hit_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache hit count - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_materialize_cache_hit_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_total_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total cached count - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_materialize_cache_total_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialize Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":53,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache lookup count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_over_window_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_cache_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache miss count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_over_window_cache_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_over_window_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache lookup count - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_over_window_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_over_window_cache_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache miss count - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_over_window_cache_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_range_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"partition range cache lookup count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_over_window_range_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_range_cache_left_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"partition range cache left miss count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_over_window_range_cache_left_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_range_cache_right_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"partition range cache right miss count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_over_window_range_cache_right_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Over Window Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":54,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, fragment_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Join executor cache miss ratio - - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} fragment {{fragment_id}}","metric":"","query":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, fragment_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n appendonly cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream lookup cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream temporal join cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Materialize executor cache miss ratio - table {{table_id}} fragment {{fragment_id}}  {{instance}}","metric":"","query":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_over_window_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Over window cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_over_window_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_over_window_range_cache_left_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_range_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Over window partition range cache left miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_over_window_range_cache_left_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_range_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_over_window_range_cache_right_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_range_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Over window partition range cache right miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_over_window_range_cache_right_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_range_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Miss Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":55,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, executor, fragment_id, wait_side, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - executor {{executor}} fragment {{fragment_id}} {{wait_side}} - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, executor, fragment_id, wait_side, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, executor, fragment_id, wait_side, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - executor {{executor}} fragment {{fragment_id}} {{wait_side}} - {{job}}","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, executor, fragment_id, wait_side, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, executor, fragment_id, wait_side, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p999 - executor {{executor}} fragment {{fragment_id}} {{wait_side}} - {{job}}","metric":"","query":"histogram_quantile(0.999, sum(rate(stream_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, executor, fragment_id, wait_side, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, executor, fragment_id, wait_side, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - executor {{executor}} fragment {{fragment_id}} {{wait_side}} - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, executor, fragment_id, wait_side, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, executor, fragment_id, wait_side, job)(rate(stream_barrier_align_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le,executor,fragment_id,wait_side,job) (rate(stream_barrier_align_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - executor {{executor}} fragment {{fragment_id}} {{wait_side}} - {{job}}","metric":"","query":"sum by(le, executor, fragment_id, wait_side, job)(rate(stream_barrier_align_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le,executor,fragment_id,wait_side,job) (rate(stream_barrier_align_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Barrier Align","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":56,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - fragment {{fragment_id}} - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - fragment {{fragment_id}} - {{job}}","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p999 - fragment {{fragment_id}} - {{job}}","metric":"","query":"histogram_quantile(0.999, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - fragment {{fragment_id}} - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, fragment_id, job)(rate(stream_merge_barrier_align_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le,fragment_id,job) (rate(stream_merge_barrier_align_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - fragment {{fragment_id}} - {{job}}","metric":"","query":"sum by(le, fragment_id, job)(rate(stream_merge_barrier_align_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le,fragment_id,job) (rate(stream_merge_barrier_align_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Merger Barrier Align","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":57,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_join_actor_input_waiting_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000) by (fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}","metric":"","query":"avg(rate(stream_join_actor_input_waiting_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000) by (fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_actor_input_waiting_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}}","metric":"","query":"rate(stream_join_actor_input_waiting_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Actor Input Blocking Time Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":58,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_join_match_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000) by (fragment_id,side)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}} {{side}}","metric":"","query":"avg(rate(stream_join_match_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000) by (fragment_id,side)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_match_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}} {{side}}","metric":"","query":"rate(stream_join_match_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Actor Match Duration Per Second","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Multiple rows with distinct primary keys may have the same join key. This metric counts the number of join keys in the executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":59,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (fragment_id, side)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}} {{side}}","metric":"","query":"sum(stream_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (fragment_id, side)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}} {{side}}","metric":"","query":"stream_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of matched rows on the opposite side","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":60,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - fragment {{fragment_id}} table_id {{table_id}} - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - fragment {{fragment_id}} table_id {{table_id}} - {{job}}","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - fragment {{fragment_id}} table_id {{table_id}} - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, fragment_id, table_id) (rate(stream_join_matched_join_keys_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, fragment_id, table_id) (rate(stream_join_matched_join_keys_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - fragment {{fragment_id}} table_id {{table_id}} - {{job}}","metric":"","query":"sum by(le, job, fragment_id, table_id) (rate(stream_join_matched_join_keys_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, fragment_id, table_id) (rate(stream_join_matched_join_keys_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) >= 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Executor Matched Rows","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":61,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_agg_chunk_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level cache miss - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_agg_chunk_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_agg_chunk_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level total lookups - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_agg_chunk_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_chunk_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level cache miss  - table {{table_id}} actor {{actor_id}}}","metric":"","query":"rate(stream_agg_chunk_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_chunk_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level total lookups  - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_chunk_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Executor Cache Statistics For Each StreamChunk","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in each hash aggregation executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":62,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_agg_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg cached keys count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_agg_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_agg_distinct_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg distinct cached keys count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_agg_distinct_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg cached keys count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_agg_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_distinct_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg distinct cached keys count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_agg_distinct_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of dirty (unflushed) groups in each hash aggregation executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":63,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_agg_dirty_groups_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_agg_dirty_groups_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_dirty_groups_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_agg_dirty_groups_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Dirty Groups Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The total heap size of dirty (unflushed) groups in each hash aggregation executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":64,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_agg_dirty_groups_heap_size{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups heap size | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_agg_dirty_groups_heap_size{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_dirty_groups_heap_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups heap size | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_agg_dirty_groups_heap_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Dirty Groups Heap Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in each top_n executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":65,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_group_top_n_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_group_top_n_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_group_top_n_appendonly_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n appendonly cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_group_top_n_appendonly_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_group_top_n_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_group_top_n_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_group_top_n_appendonly_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n appendonly cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_group_top_n_appendonly_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"TopN Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in temporal join executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":66,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_temporal_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal Join cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_temporal_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_temporal_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal Join cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_temporal_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Temporal Join Cache Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in lookup executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":88},"height":null,"hideTimeOverride":false,"id":67,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_lookup_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lookup cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_lookup_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_lookup_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lookup cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_lookup_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lookup Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in over window executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":88},"height":null,"hideTimeOverride":false,"id":68,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_over_window_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"over window cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_over_window_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_over_window_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"over window cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_over_window_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_over_window_range_cache_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"over window partition range cache entry count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_over_window_range_cache_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Over Window Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"When enabled, this metric shows the input throughput of each executor.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":96},"height":null,"hideTimeOverride":false,"id":69,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_executor_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_identity, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_identity}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_executor_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_identity, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_executor_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_identity}} actor {{actor_id}}","metric":"","query":"rate(stream_executor_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The actor-level memory usage statistics reported by TaskLocalAlloc. (Disabled by default)","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":96},"height":null,"hideTimeOverride":false,"id":70,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(actor_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}","metric":"","query":"sum(actor_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"actor_memory_usage{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}}","metric":"","query":"actor_memory_usage{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Memory Usage (TaskLocalAlloc)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Actors","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":6},"height":null,"hideTimeOverride":false,"id":71,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":72,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_actor_execution_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_actor_execution_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Execution Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":73,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":74,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":8},"height":null,"hideTimeOverride":false,"id":75,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":8},"height":null,"hideTimeOverride":false,"id":76,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":77,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":16},"height":null,"hideTimeOverride":false,"id":78,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":16},"height":null,"hideTimeOverride":false,"id":79,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":80,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":24},"height":null,"hideTimeOverride":false,"id":81,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":24},"height":null,"hideTimeOverride":false,"id":82,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":83,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":32},"height":null,"hideTimeOverride":false,"id":84,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":32},"height":null,"hideTimeOverride":false,"id":85,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":86,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":40},"height":null,"hideTimeOverride":false,"id":87,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Avg Time","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Actors (Tokio)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":7},"height":null,"hideTimeOverride":false,"id":88,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":89,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{up_fragment_id}}->{{down_fragment_id}}","metric":"","query":"rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fragment-level Remote Exchange Send Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":90,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{up_fragment_id}}->{{down_fragment_id}}","metric":"","query":"rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fragment-level Remote Exchange Recv Throughput","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Exchange","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":91,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Errors that happened during computation. Check the logs for detailed error message.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":92,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_compute_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, executor_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{executor_name}} (fragment_id={{fragment_id}})","metric":"","query":"sum(user_compute_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, executor_name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compute Errors by Type","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Errors that happened during source data ingestion. Check the logs for detailed error message.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":93,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_source_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{source_name}} (source_id={{source_id}} fragment_id={{fragment_id}})","metric":"","query":"sum(user_source_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, source_id, source_name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Errors by Type","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Errors that happened during data sink out. Check the logs for detailed error message.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":94,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_sink_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, sink_id, sink_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{sink_name}} (sink_id={{sink_id}} fragment_id={{fragment_id}})","metric":"","query":"sum(user_sink_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, sink_id, sink_name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Errors by Type","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"User Streaming Errors","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":9},"height":null,"hideTimeOverride":false,"id":95,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"row"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":96,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{query_id}} : {{source_stage_id}}.{{source_task_id}} -> {{target_stage_id}}.{{target_task_id}}","metric":"","query":"batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Exchange Recv Row Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":97,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_task_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"batch_task_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Mpp Task Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"All memory usage of batch executors in bytes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":98,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"compute_batch_total_mem{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"compute_batch_total_mem{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"frontend_batch_total_mem{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"frontend_batch_total_mem{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Mem Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":99,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_heartbeat_worker_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"batch_heartbeat_worker_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Heartbeat Worker Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":100,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Row SeqScan Next Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Disk throughputs of spilling-out in the bacth query engine","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":101,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(batch_spill_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}} @ {{instance}}","metric":"","query":"sum(rate(batch_spill_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(batch_spill_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}} @ {{instance}}","metric":"","query":"sum(rate(batch_spill_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Spill Throughput","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Batch Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":10},"height":null,"hideTimeOverride":false,"id":102,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":103,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{table_id}} @ {{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_sst_store_block_request_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total_meta_miss_count - {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Hummock has three parts of memory usage: 1. Meta Cache 2. Block CacheThis metric shows the real memory usage of each of these three caches.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":104,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta cache - {{job}} @ {{instance}}","metric":"","query":"avg(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"data cache - {{job}} @ {{instance}}","metric":"","query":"avg(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(state_store_prefetch_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"prefetch cache - {{job}} @ {{instance}}","metric":"","query":"avg(state_store_prefetch_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":105,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='meta_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta cache miss ratio - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"(sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='meta_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_sst_store_block_request_counts{type='data_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='data_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache miss ratio - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"(sum(rate(state_store_sst_store_block_request_counts{type='data_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='data_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) >= 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Miss Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of the estimated hit ratio of a block while in the block cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":106,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_max(histogram_quantile(0.1, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache efficienfy - p10 - {{job}} @ {{instance}}","metric":"","query":"clamp_max(histogram_quantile(0.1, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_max(histogram_quantile(0.25, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache efficienfy - p25 - {{job}} @ {{instance}}","metric":"","query":"clamp_max(histogram_quantile(0.25, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_max(histogram_quantile(0.5, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache efficienfy - p50 - {{job}} @ {{instance}}","metric":"","query":"clamp_max(histogram_quantile(0.5, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_max(histogram_quantile(0.75, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache efficienfy - p75 - {{job}} @ {{instance}}","metric":"","query":"clamp_max(histogram_quantile(0.75, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_max(histogram_quantile(0.9, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache efficienfy - p90 - {{job}} @ {{instance}}","metric":"","query":"clamp_max(histogram_quantile(0.9, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_max(histogram_quantile(1.0, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache efficienfy - p100 - {{job}} @ {{instance}}","metric":"","query":"clamp_max(histogram_quantile(1.0, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Block Cache Efficiency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":107,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_scan_key_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type, table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter keys flow - {{table_id}} @ {{type}} @ {{instance}}","metric":"","query":"sum(rate(state_store_iter_scan_key_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type, table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iter keys flow","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":108,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts p50 - {{table_id}} @ {{job}} @ {{type}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts p99 - {{table_id}} @ {{job}} @ {{type}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts pmax - {{table_id}} @ {{job}} @ {{type}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts avg  - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Merged SSTs","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of the latency of Get operations that have been issued to the state store.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":109,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_get_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, table_id)(rate(state_store_get_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Duration - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of the time spent on iterator initialization.Histogram of the time spent on iterator scanning.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":110,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time p50 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time p99 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time pmax - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_iter_init_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, iter_type) (rate(state_store_iter_init_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time avg - {{iter_type}} {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(state_store_iter_init_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, iter_type) (rate(state_store_iter_init_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time p50 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time p99 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time pmax - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_iter_scan_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, iter_type) (rate(state_store_iter_scan_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time avg - {{iter_type}} {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(state_store_iter_scan_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, iter_type) (rate(state_store_iter_scan_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Duration - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":111,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter false positive count  - {{table_id}} - {{type}}","metric":"","query":"sum(irate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter positive count - {{table_id}} - {{type}}","metric":"","query":"sum(irate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter check count- {{table_id}} - {{type}}","metric":"","query":"sum(irate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom Filter Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Positive / Total","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":112,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter positive rate - {{table_id}} - {{type}}","metric":"","query":"(sum(rate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)) >= 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom Filter Positive Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"False-Positive / Total","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":113,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(((sum(rate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read req bloom filter false positive rate - {{table_id}} - {{type}}","metric":"","query":"(((sum(rate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))) >= 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom Filter False-Positive Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":114,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_iter_slow_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"state_store_iter_slow_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Slow Fetch Meta Unhits","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":115,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_shared_buffer_hit_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"shared_buffer hit - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_get_shared_buffer_hit_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id, iter_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{iter_type}} - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_iter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id, iter_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":116,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Size - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":117,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Size - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":118,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read p50 - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read p99 - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read pmax - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Read Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":119,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_iter_in_progress_counts{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Existing {{iter_type}} count @ {{table_id}}","metric":"","query":"state_store_iter_in_progress_counts{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_log_op_type_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, op_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter_log op count @ {{table_id}} {{op_type}}","metric":"","query":"sum(rate(state_store_iter_log_op_type_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, op_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Count - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size of a single key-value pair when reading by operation Get.Operation Get gets a single key-value pair with respect to a caller-specified key. If the key does not exist in the storage, the size of key is counted into this metric and the size of value is 0.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":120,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance) + sum(rate(state_store_get_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_get_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance) + sum(rate(state_store_get_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Throughput - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size of all the key-value paris when reading by operation Iter.Operation Iter scans a range of key-value pairs.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":121,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_iter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Throughput - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":122,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration avg - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fetch Meta Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":123,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_iter_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"state_store_iter_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fetch Meta Unhits","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock (Read)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":11},"height":null,"hideTimeOverride":false,"id":124,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric shows the real memory usage of uploader.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":125,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploading memory - {{job}} @ {{instance}}","metric":"","query":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_uploading_task_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploading task size - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_uploader_uploading_task_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_imm_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploader imm size - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_uploader_imm_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_imm_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance) - sum(state_store_uploader_uploading_task_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unflushed imm size - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_uploader_imm_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance) - sum(state_store_uploader_uploading_task_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance) - sum(state_store_uploader_imm_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"orphan imm size - {{job}} @ {{instance}}","metric":"","query":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance) - sum(state_store_uploader_imm_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_old_value_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"old value size - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_old_value_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader Memory Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of time spent on compacting shared buffer to remote storage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":126,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 Sync duration - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 Sync duration - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax Sync duration - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_sync_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg Sync duration - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance) (rate(state_store_sync_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_uploader_upload_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 upload task duration - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_uploader_upload_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_uploader_upload_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 upload task duration - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_uploader_upload_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_uploader_upload_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax upload task duration - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_uploader_upload_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Build and Sync Sstable Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":127,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.5, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write p50 - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(0.5, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.99, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write p99 - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(0.99, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write pmax - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Write Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":128,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_merge_imm_task_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"merge imm tasks - {{table_id}} @ {{instance}}","metric":"","query":"sum(irate(state_store_merge_imm_task_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_spill_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Uploader spill tasks - {{uploader_stage}} @ {{instance}}","metric":"","query":"sum(irate(state_store_spill_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_uploading_task_count{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploading task count - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_uploader_uploading_task_count{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_syncing_epoch_count{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"syncing epoch count - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_uploader_syncing_epoch_count{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader - Tasks Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":129,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_merge_imm_memory_sz{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Merging tasks memory size - {{table_id}} @ {{instance}}","metric":"","query":"sum(rate(state_store_merge_imm_memory_sz{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_spill_task_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Uploading tasks size - {{uploader_stage}} @ {{instance}}","metric":"","query":"sum(rate(state_store_spill_task_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader - Task Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":130,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write batch - {{table_id}} @ {{job}} @ {{instance}} ","metric":"","query":"sum(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"l0 - {{job}} @ {{instance}} ","metric":"","query":"sum(rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":131,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer avg - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":132,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_write_batch_tuple_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write_batch_kv_pair_count - {{table_id}} @ {{instance}}","metric":"","query":"sum(irate(state_store_write_batch_tuple_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Item Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":133,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_write_batch_size_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id) / sum(rate(state_store_write_batch_size_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"shared_buffer - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_write_batch_size_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id) / sum(rate(state_store_write_batch_size_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_shared_buffer_to_sstable_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance) / sum(rate(compactor_shared_buffer_to_sstable_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sync - {{job}} @ {{instance}}","metric":"","query":"sum(rate(compactor_shared_buffer_to_sstable_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance) / sum(rate(compactor_shared_buffer_to_sstable_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric shows the statistics of mem_table size on flush. By default only max (p100) is shown.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":134,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_write_batch_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, table_id, job, instance) (rate(state_store_write_batch_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance) (rate(state_store_write_batch_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, table_id, job, instance) (rate(state_store_write_batch_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Batch Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":135,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_mem_table_spill_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mem table spill table id - {{table_id}} @ {{instance}}","metric":"","query":"sum(irate(state_store_mem_table_spill_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Mem Table Spill Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":136,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Checkpoint Sync Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":137,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_event_handler_pending_event{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"sum(state_store_event_handler_pending_event{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Event handler pending event number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":138,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_event_handler_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, event_type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 {{event_type}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_event_handler_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, event_type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_event_handler_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, event_type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 {{event_type}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_event_handler_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, event_type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_event_handler_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, event_type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax {{event_type}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_event_handler_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, event_type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_uploader_wait_poll_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 finished_task_wait_poll {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_uploader_wait_poll_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_uploader_wait_poll_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 finished_task_wait_poll {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_uploader_wait_poll_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_uploader_wait_poll_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax finished_task_wait_poll {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_uploader_wait_poll_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Event handle latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock (Write)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":12},"height":null,"hideTimeOverride":false,"id":139,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of SSTables at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":140,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","query":"sum(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"SSTable Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size(KB) of SSTables at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":141,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","query":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"SSTable Size(KB)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The  of bytes that have been written by commit epoch per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":142,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_commit_write_throughput{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{table_id}}","metric":"","query":"sum(rate(storage_commit_write_throughput{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Commit Flush Bytes by Table","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have completed or failed","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":143,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_frequency{result!='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task_type}} - {{result}} - group-{{group}} @ {{compactor}}","metric":"","query":"sum(storage_level_compact_frequency{result!='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Failure Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have completed or failed","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":144,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_frequency{result='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task_type}} - {{result}} - group-{{group}} @ {{compactor}}","metric":"","query":"sum(storage_level_compact_frequency{result='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Success Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have been skipped.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":145,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_skip_compact_frequency{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (level, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{level}}-{{type}}","metric":"","query":"sum(rate(storage_skip_compact_frequency{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (level, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Skip Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Avg l0 select_level_count of the compact task, and categorize it according to different cg, levels and task types","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":146,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, group, type)(irate(storage_l0_compact_level_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, group, type)(irate(storage_l0_compact_level_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg cg{{group}}@{{type}}","metric":"","query":"sum by(le, group, type)(irate(storage_l0_compact_level_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, group, type)(irate(storage_l0_compact_level_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task L0 Select Level Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Avg file count of the compact task, and categorize it according to different cg, levels and task types","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":147,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, group, type)(irate(storage_compact_task_file_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, group, type)(irate(storage_compact_task_file_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg cg{{group}}@{{type}}","metric":"","query":"sum by(le, group, type)(irate(storage_compact_task_file_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, group, type)(irate(storage_compact_task_file_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task File Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The distribution of the compact task size triggered, including p90 and max. and categorize it according to different cg, levels and task types.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":148,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - cg{{group}}@{{type}}","metric":"","query":"histogram_quantile(0.9, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - cg{{group}}@{{type}}","metric":"","query":"histogram_quantile(1.0, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task Size Distribution","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that are running.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":149,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(storage_compact_task_pending_num{job=~\"$job\",instance=~\"$node\"}) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor_task_count - {{job}} @ {{instance}}","metric":"","query":"avg(storage_compact_task_pending_num{job=~\"$job\",instance=~\"$node\"}) by(job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(storage_compact_task_pending_parallelism{job=~\"$job\",instance=~\"$node\"}) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor_task_pending_parallelism - {{job}} @ {{instance}}","metric":"","query":"avg(storage_compact_task_pending_parallelism{job=~\"$job\",instance=~\"$node\"}) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compactor Running Task Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"compact-task: The total time have been spent on compaction.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":150,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task p50 - {{job}}","metric":"","query":"histogram_quantile(0.5, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task p90 - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task pmax - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range p90 - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range pmax - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get-table-id p90 - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get-table-id pmax - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io p90 - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io pmax - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(compute_refill_cache_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compute_apply_version_duration_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(compute_refill_cache_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le)(rate(compactor_compact_task_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(compactor_compact_task_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task avg","metric":"","query":"sum by(le)(rate(compactor_compact_task_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(compactor_compact_task_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le)(rate(state_store_compact_sst_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(state_store_compact_sst_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range avg","metric":"","query":"sum by(le)(rate(state_store_compact_sst_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(state_store_compact_sst_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"KBs read from next level during history compactions to next level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":151,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job) + sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}}","metric":"","query":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job) + sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}}","metric":"","query":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"flush - {{job}}","metric":"","query":"sum(rate(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_fast_compact_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fast compact - {{job}}","metric":"","query":"sum(rate(compactor_fast_compact_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of bytes that have been written by compaction.Flush refers to the process of compacting Memtables to SSTables at Level 0.Write refers to the process of compacting SSTables at one level to another level.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":152,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}}","metric":"","query":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"flush - {{job}}","metric":"","query":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Write Bytes(GiB)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Write amplification is the amount of bytes written to the remote storage by compaction for each one byte of flushed SSTable data. Write amplification is by definition higher than 1.0 because we write each piece of data to L0, and then write it again to an SSTable, and then compaction may read this piece of data and write it to a new SSTable, that's another write.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":153,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) / sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write amplification","metric":"","query":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) / sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Write Amplification","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of SSTables that is being compacted at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":154,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_level_compact_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","query":"storage_level_compact_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compacting SSTable Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"num of compact_task","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":155,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_level_compact_task_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task}}","metric":"","query":"storage_level_compact_task_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compacting Task Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":156,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from next level","metric":"","query":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from current level","metric":"","query":"sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} write to next level","metric":"","query":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"KBs Read/Write by Level","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":157,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_write_sstn{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} write to next level","metric":"","query":"sum(irate(storage_level_compact_write_sstn{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_read_sstn_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from next level","metric":"","query":"sum(irate(storage_level_compact_read_sstn_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_read_sstn_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from current level","metric":"","query":"sum(irate(storage_level_compact_read_sstn_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Count of SSTs Read/Write by level","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total bytes gotten from sstable_bloom_filter, for observing bloom_filter size","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":158,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_bloom_filter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_bloom_filter_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_meta - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_bloom_filter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_bloom_filter_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_file_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_file_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_file - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_file_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_file_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total bytes gotten from sstable_avg_key_size, for observing sstable_avg_key_size","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":159,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_key_size - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_value_size - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Item Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Avg count gotten from sstable_distinct_epoch_count, for observing sstable_distinct_epoch_count","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":160,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_distinct_epoch_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_distinct_epoch_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_epoch_count - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_distinct_epoch_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_distinct_epoch_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Stat","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total time of operations which read from remote storage when enable prefetch","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":161,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io p90 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Remote Read Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":88},"height":null,"hideTimeOverride":false,"id":162,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_iter_scan_key_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter keys flow - {{type}} @ {{instance}} ","metric":"","query":"sum(rate(compactor_iter_scan_key_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compactor Iter keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"bytes of Lsm tree needed to reach balance","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":88},"height":null,"hideTimeOverride":false,"id":163,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_compact_pending_bytes{job=~\"$job\",instance=~\"$node\"}) by (instance, group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact pending bytes - {{group}} @ {{instance}} ","metric":"","query":"sum(storage_compact_pending_bytes{job=~\"$job\",instance=~\"$node\"}) by (instance, group)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lsm Compact Pending Bytes","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"compression ratio of each level of the lsm tree","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":96},"height":null,"hideTimeOverride":false,"id":164,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_compact_level_compression_ratio{job=~\"$job\",instance=~\"$node\"}) by (instance, group, level, algorithm)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lsm compression ratio - cg{{group}} @ L{{level}} - {{algorithm}} {{instance}}","metric":"","query":"sum(storage_compact_level_compression_ratio{job=~\"$job\",instance=~\"$node\"}) by (instance, group, level, algorithm)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lsm Level Compression Ratio","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Compaction","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":13},"height":null,"hideTimeOverride":false,"id":165,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":166,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":167,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, type, job, instance)(rate(object_store_operation_latency_sum{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(object_store_operation_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, type, job, instance)(rate(object_store_operation_latency_sum{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(object_store_operation_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":168,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type!~'streaming_upload_write_bytes|streaming_read_read_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_operation_latency_count{type!~'streaming_upload_write_bytes|streaming_read_read_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type=~'upload|delete',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{media_type}}-write - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_operation_latency_count{type=~'upload|delete',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type=~'read|readv|list|metadata',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{media_type}}-read - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_operation_latency_count{type=~'read|readv|list|metadata',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":169,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":170,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Failure Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":171,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_request_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_request_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Retry Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"There are two types of operations: 1. GET, SELECT, and DELETE, they cost 0.0004 USD per 1000 requests. 2. PUT, COPY, POST, LIST, they cost 0.005 USD per 1000 requests.Reading from S3 across different regions impose extra cost. This metric assumes 0.01 USD per 1GB data transfer. Please checkout AWS's pricing model for more accurate calculation.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"$"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":172,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}) * 0.01 / 1000 / 1000 / 1000","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"(Cross Region) Data Transfer Cost","metric":"","query":"sum(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}) * 0.01 / 1000 / 1000 / 1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_operation_latency_count{type=~'read|streaming_read_start|streaming_read_init',job=~\"$job\",instance=~\"$node\"}) * 0.0004 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GET, SELECT, and all other Requests Cost","metric":"","query":"sum(object_store_operation_latency_count{type=~'read|streaming_read_start|streaming_read_init',job=~\"$job\",instance=~\"$node\"}) * 0.0004 / 1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_operation_latency_count{type=~'upload|streaming_upload|streaming_upload_start|s3_upload_part|streaming_upload_finish|list',job=~\"$job\",instance=~\"$node\"}) * 0.005 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"PUT, COPY, POST, LIST Requests Cost","metric":"","query":"sum(object_store_operation_latency_count{type=~'upload|streaming_upload|streaming_upload_start|s3_upload_part|streaming_upload_finish|list',job=~\"$job\",instance=~\"$node\"}) * 0.005 / 1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Estimated S3 Cost (Realtime)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric uses the total size of data in S3 at this second to derive the cost of storing data for a whole month. The price is 0.023 USD per GB. Please checkout AWS's pricing model for more accurate calculation.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"$"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":173,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance) * 0.023 / 1000 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Monthly Storage Cost","metric":"","query":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance) * 0.023 / 1000 / 1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Estimated S3 Cost (Monthly)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Object Storage","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":14},"height":null,"hideTimeOverride":false,"id":174,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":175,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_hybrid_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - hybrid - {{op}} @ {{instance}}","metric":"","query":"sum(rate(foyer_hybrid_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hybrid Cache Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":176,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{name}} - hybrid - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{name}} - hybrid - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{name}} - hybrid - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{name}} - hybrid - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hybrid Cache Op Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":177,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_hybrid_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) / (sum(rate(foyer_hybrid_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) + sum(rate(foyer_hybrid_op_total{op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - hybrid - hit ratio @ {{instance}}","metric":"","query":"sum(rate(foyer_hybrid_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) / (sum(rate(foyer_hybrid_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) + sum(rate(foyer_hybrid_op_total{op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hybrid Cache Hit Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":178,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_memory_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - memory - {{op}} @ {{instance}}","metric":"","query":"sum(rate(foyer_memory_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Cache Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":179,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(foyer_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (name, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - memory - size @ {{instance}}","metric":"","query":"sum(foyer_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (name, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Cache Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":180,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_memory_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) / (sum(rate(foyer_memory_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) + sum(rate(foyer_memory_op_total{op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - memory - hit ratio @ {{instance}}","metric":"","query":"sum(rate(foyer_memory_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) / (sum(rate(foyer_memory_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) + sum(rate(foyer_memory_op_total{op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Cache Hit Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":181,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"sum(rate(foyer_storage_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":182,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_inner_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"sum(rate(foyer_storage_inner_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache Inner Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":183,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache Op Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":184,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache Inner Op Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":185,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) / (sum(rate(foyer_storage_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) + sum(rate(foyer_storage_op_total{op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - storage - hit ratio @ {{instance}}","metric":"","query":"sum(rate(foyer_storage_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) / (sum(rate(foyer_storage_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) + sum(rate(foyer_storage_op_total{op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache Hit Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":186,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(foyer_storage_region{job=~\"$job\",instance=~\"$node\"}) by (name, type, instance) * on(name, instance) group_left() avg(foyer_storage_region_size_bytes{job=~\"$job\",instance=~\"$node\"}) by (name, type, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - {{type}} region - size @ {{instance}}","metric":"","query":"sum(foyer_storage_region{job=~\"$job\",instance=~\"$node\"}) by (name, type, instance) * on(name, instance) group_left() avg(foyer_storage_region_size_bytes{job=~\"$job\",instance=~\"$node\"}) by (name, type, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Region Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":187,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_disk_io_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - disk - {{op}} @ {{instance}}","metric":"","query":"sum(rate(foyer_storage_disk_io_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Disk Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":188,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{name}} - disk - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{name}} - disk - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{name}} - disk - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{name}} - disk - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Disk Op Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":189,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_disk_io_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - disk - {{op}} @ {{instance}}","metric":"","query":"sum(rate(foyer_storage_disk_io_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Disk Op Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":190,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (type, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} file cache refill - {{op}} @ {{instance}}","metric":"","query":"sum(rate(refill_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (type, op, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=~\"meta|data\",op!~\"filtered|ignored\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (type, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} file cache refill - {{op}} @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=~\"meta|data\",op!~\"filtered|ignored\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (type, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":191,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} file cache - {{op}} @ {{instance}}","metric":"","query":"sum(rate(refill_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Data Refill Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":192,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{foyer}} cache refill - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{foyer}} cache refill - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{foyer}} cache refill - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{foyer}} cache refill - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":193,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(refill_queue_total) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"refill queue length @ {{instance}}","metric":"","query":"sum(refill_queue_total) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill Queue Length","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":194,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"parent_meta\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"parent meta lookup {{op}} @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=\"parent_meta\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inheritance - Parent Meta Lookup Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":195,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"parent_meta\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / (sum(rate(refill_total{type=\"parent_meta\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) + sum(rate(refill_total{type=\"parent_meta\",op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"parent meta lookup hit ratio @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=\"parent_meta\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / (sum(rate(refill_total{type=\"parent_meta\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) + sum(rate(refill_total{type=\"parent_meta\",op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)) >= 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inheritance - Parent Meta Lookup Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":196,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"unit_inheritance\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unit inheritance {{op}} @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=\"unit_inheritance\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inheritance - Unit inheritance Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":88},"height":null,"hideTimeOverride":false,"id":197,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"unit_inheritance\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / (sum(rate(refill_total{type=\"unit_inheritance\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) + sum(rate(refill_total{type=\"unit_inheritance\",op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unit inheritance ratio @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=\"unit_inheritance\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / (sum(rate(refill_total{type=\"unit_inheritance\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) + sum(rate(refill_total{type=\"unit_inheritance\",op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)) >= 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inheritance - Unit inheritance Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":88},"height":null,"hideTimeOverride":false,"id":198,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"block\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block refill {{op}} @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=\"block\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Block Refill Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":96},"height":null,"hideTimeOverride":false,"id":199,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"block\",op=\"success\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / sum(rate(refill_total{type=\"block\",op=\"unfiltered\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block refill ratio @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=\"block\",op=\"success\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / sum(rate(refill_total{type=\"block\",op=\"unfiltered\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) >= 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Block Refill Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Item numbers of the recent filter.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":96},"height":null,"hideTimeOverride":false,"id":200,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(recent_filter_items{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"items @ {{instance}}","metric":"","query":"sum(recent_filter_items{job=~\"$job\",instance=~\"$node\"}) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Recent Filter Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":104},"height":null,"hideTimeOverride":false,"id":201,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(recent_filter_ops{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recent filter {{op}} @ {{instance}}","metric":"","query":"sum(rate(recent_filter_ops{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Recent Filter Ops","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock Tiered Cache","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":15},"height":null,"hideTimeOverride":false,"id":202,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":203,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time p50 - {{method}} @ {{lock_type}} @ {{lock_name}}","metric":"","query":"histogram_quantile(0.5, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method, lock_name, lock_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time p99 - {{method}} @ {{lock_type}} @ {{lock_name}}","metric":"","query":"histogram_quantile(0.99, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method, lock_name, lock_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time pmax - {{method}} @ {{lock_type}} @ {{lock_name}}","metric":"","query":"histogram_quantile(1.0, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method, lock_name, lock_type))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lock Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":204,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time p50 - {{method}}","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time p99 - {{method}}","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time pmax - {{method}}","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Real Process Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":205,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version size","metric":"","query":"storage_version_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":206,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"current version id","metric":"","query":"storage_current_version_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"checkpoint version id","metric":"","query":"storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min pinned version id","metric":"","query":"storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_safepoint_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min safepoint version id","metric":"","query":"storage_min_safepoint_version_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Id","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":207,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"max committed epoch","metric":"","query":"storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_safe_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"safe epoch","metric":"","query":"storage_safe_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min pinned epoch","metric":"","query":"storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Epoch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":208,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_key_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","query":"storage_version_stats{metric='total_key_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_value_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","query":"storage_version_stats{metric='total_value_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Table Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":209,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_materialized_view_stats{metric='materialized_view_total_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{metric}}, mv id - {{table_id}} ","metric":"","query":"storage_materialized_view_stats{metric='materialized_view_total_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":210,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_key_count',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","query":"storage_version_stats{metric='total_key_count',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Table KV Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"\nObjects are classified into 3 groups:\n- not referenced by versions: these object are being deleted from object store.\n- referenced by non-current versions: these objects are stale (not in the latest version), but those old versions may still be in use (e.g. long-running pinning). Thus those objects cannot be deleted at the moment.\n- referenced by current version: these objects are in the latest version.\n\nAdditionally, a metric on all objects (including dangling ones) is updated with low-frequency. The metric is updated right before full GC. So subsequent full GC may reduce the actual value significantly, without updating the metric.\n                    ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":211,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","query":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","query":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","query":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_total_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all objects (including dangling ones)","metric":"","query":"storage_total_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Refer to `Object Total Number` panel for classification of objects.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":212,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","query":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","query":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","query":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_total_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all objects, including dangling ones","metric":"","query":"storage_total_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"total number of hummock version delta log","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":213,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_delta_log_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"delta log total number","metric":"","query":"storage_delta_log_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Delta Log Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"hummock version checkpoint latency","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":214,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(storage_version_checkpoint_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(storage_version_checkpoint_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_avg","metric":"","query":"rate(storage_version_checkpoint_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(storage_version_checkpoint_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Checkpoint Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"When certain per compaction group threshold is exceeded (e.g. number of level 0 sub-level in LSMtree), write op to that compaction group is stopped temporarily. Check log for detail reason of write stop.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":215,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_write_stop_compaction_groups{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compaction_group_{{compaction_group_id}}","metric":"","query":"storage_write_stop_compaction_groups{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Stop Compaction Groups","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"total number of attempts to trigger full GC","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":216,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_full_gc_trigger_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"full_gc_trigger_count","metric":"","query":"storage_full_gc_trigger_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Full GC Trigger Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"the object id watermark used in last full GC","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":217,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_full_gc_last_object_id_watermark{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"full_gc_last_object_id_watermark","metric":"","query":"storage_full_gc_last_object_id_watermark{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Full GC Last Watermark","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":218,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Event Loop Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The times of move_state_table occurs","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":219,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_move_state_table_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"move table cg{{group}}","metric":"","query":"sum(storage_move_state_table_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}) by (group)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Move State Table Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of state_tables in each CG","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":220,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_state_table_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"state table cg{{group}}","metric":"","query":"sum(irate(storage_state_table_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"State Table Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of branched_sst in each CG","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":221,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_branched_sst_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"branched sst cg{{group}}","metric":"","query":"sum(irate(storage_branched_sst_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Branched SST Count","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":222,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total backup job count since the Meta node starts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":223,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"backup_job_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"job count","metric":"","query":"backup_job_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Job Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Latency of backup jobs since the Meta node starts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":224,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time p50 - {{state}}","metric":"","query":"histogram_quantile(0.5, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time p99 - {{state}}","metric":"","query":"histogram_quantile(0.99, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time pmax - {{state}}","metric":"","query":"histogram_quantile(1.0, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Job Process Time","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Backup Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":17},"height":null,"hideTimeOverride":false,"id":225,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":226,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":227,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Drop latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":228,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"GetCatalog latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Catalog Service","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":18},"height":null,"hideTimeOverride":false,"id":229,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":230,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"AddWorkerNode latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":231,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"ListAllNodes latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Cluster Service","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":19},"height":null,"hideTimeOverride":false,"id":232,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":233,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CreateMaterializedView latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":234,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"DropMaterializedView latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":235,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Flush latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Stream Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":20},"height":null,"hideTimeOverride":false,"id":236,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":237,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UnpinVersionBefore latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":238,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UnpinSnapshotBefore latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":239,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"ReportCompactionTasks latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":240,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"GetNewSstIds latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Hummock Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":21},"height":null,"hideTimeOverride":false,"id":241,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":242,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_report_compaction_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_counts - {{instance}}","metric":"","query":"sum(irate(state_store_report_compaction_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"compaction_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":243,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_version_before_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_version_before_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_avg","metric":"","query":"sum(irate(state_store_unpin_version_before_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_version_before_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"version_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":244,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latencyp90 - {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_pin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_pin_snapshot_latency_count[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_avg","metric":"","query":"sum(irate(state_store_pin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_pin_snapshot_latency_count[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_snapshot_latency_count[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_avg","metric":"","query":"sum(irate(state_store_unpin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_snapshot_latency_count[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_unpin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(irate(state_store_unpin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"snapshot_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":245,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_pin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_counts - {{instance}}","metric":"","query":"sum(irate(state_store_pin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_counts - {{instance}}","metric":"","query":"sum(irate(state_store_unpin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"snapshot_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":246,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_get_new_sst_ids_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_get_new_sst_ids_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_avg","metric":"","query":"sum(irate(state_store_get_new_sst_ids_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_get_new_sst_ids_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"table_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":247,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_get_new_sst_ids_latency_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_counts - {{instance}}","metric":"","query":"sum(irate(state_store_get_new_sst_ids_latency_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"table_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":248,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_report_compaction_task_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_report_compaction_task_latency_count[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_avg","metric":"","query":"sum(irate(state_store_report_compaction_task_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_report_compaction_task_latency_count[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"compaction_latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC: Hummock Meta Client","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":22},"height":null,"hideTimeOverride":false,"id":249,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of active sessions","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":250,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Active Sessions","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":251,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Per Second (Local Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":252,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Per Second (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":253,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of running query in distributed execution mode","metric":"","query":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Running Queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":254,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of rejected query in distributed execution mode","metric":"","query":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Rejected queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":255,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of completed query in distributed execution mode","metric":"","query":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Completed Queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":256,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":257,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency (Local Query Mode)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Frontend","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":23},"height":null,"hideTimeOverride":false,"id":258,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":259,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(lru_runtime_loop_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"rate(lru_runtime_loop_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager loop count per sec","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":260,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_eviction_policy{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"lru_eviction_policy{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager eviction policy","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":261,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_latest_sequence{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"lru_latest_sequence{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_watermark_sequence{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"lru_watermark_sequence{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager sequence","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":262,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jemalloc_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The allocated memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":263,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_active_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jemalloc_active_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The active memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":264,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_resident_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jemalloc_resident_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The resident memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":265,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_metadata_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jemalloc_metadata_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The metadata memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":266,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jvm_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jvm_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The allocated memory of jvm","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":267,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jvm_active_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jvm_active_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The active memory of jvm","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":268,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_current_watermark_time_ms{job=~\"$job\",instance=~\"$node\"} - on() group_right() lru_evicted_watermark_time_ms{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table {{table_id}} actor {{actor_id}} desc: {{desc}}","metric":"","query":"lru_current_watermark_time_ms{job=~\"$job\",instance=~\"$node\"} - on() group_right() lru_evicted_watermark_time_ms{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager diff between current watermark and evicted watermark time (ms) for actors","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Memory manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":269,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":270,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(source_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_type}} @ {{source_id}}","metric":"","query":"rate(source_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Source Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":271,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(connector_sink_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector_type}} @ {{sink_id}}","metric":"","query":"rate(connector_sink_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Sink Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Connector Node","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":25},"height":null,"hideTimeOverride":false,"id":272,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":273,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 @ {{connector}} {{sink_id}}","metric":"","query":"histogram_quantile(0.5, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 @ {{connector}} {{sink_id}}","metric":"","query":"histogram_quantile(0.99, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax @ {{connector}} {{sink_id}}","metric":"","query":"histogram_quantile(1.0, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, connector, sink_id)(rate(sink_commit_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(sink_commit_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{connector}} @ {{sink_id}}","metric":"","query":"sum by(le, connector, sink_id)(rate(sink_commit_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(sink_commit_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Commit Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":274,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"log_store_latest_write_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"latest write epoch @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","query":"log_store_latest_write_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"latest read epoch @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","query":"log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"kv_log_store_buffer_unconsumed_min_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Kv log store uncomsuned min epoch @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","query":"kv_log_store_buffer_unconsumed_min_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Read/Write Epoch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":275,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(max(log_store_latest_write_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)- max(log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)) / (2^16) / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Consume lag @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","query":"(max(log_store_latest_write_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)- max(log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)) / (2^16) / 1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Lag","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":276,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(log_store_reader_wait_new_future_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id, executor_id) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Backpressure @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","query":"avg(rate(log_store_reader_wait_new_future_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id, executor_id) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Backpressure Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":277,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_min((max(log_store_first_write_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)- max(log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)) / (2^16) / 1000, 0)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Consume persistent log lag @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","query":"clamp_min((max(log_store_first_write_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)- max(log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)) / (2^16) / 1000, 0)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Consume Persistent Log Lag","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":278,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_read_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector}} {{sink_id}}","metric":"","query":"sum(rate(log_store_read_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Consume Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":279,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_read_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, connector, sink_id, executor_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector}} {{sink_id}} @ {{executor_id}} {{instance}}","metric":"","query":"sum(rate(log_store_read_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, connector, sink_id, executor_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Log Store Consume Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":280,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_write_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector}} {{sink_id}}","metric":"","query":"sum(rate(log_store_write_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Write Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":281,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_write_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, connector, sink_id, executor_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector}} {{sink_id}} @ {{executor_id}} {{instance}}","metric":"","query":"sum(rate(log_store_write_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, connector, sink_id, executor_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Log Store Write Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":282,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_read_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","query":"sum(rate(kv_log_store_storage_read_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Read Storage Row Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":283,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_read_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","query":"sum(rate(kv_log_store_storage_read_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Read Storage Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":284,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_write_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","query":"sum(rate(kv_log_store_storage_write_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Write Storage Row Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":285,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_write_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","query":"sum(rate(kv_log_store_storage_write_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Write Storage Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":286,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"kv_log_store_buffer_unconsumed_item_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Unconsumed item count @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","query":"kv_log_store_buffer_unconsumed_item_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"kv_log_store_buffer_unconsumed_row_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Unconsumed row count @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","query":"kv_log_store_buffer_unconsumed_row_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"kv_log_store_buffer_unconsumed_epoch_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Unconsumed epoch count @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","query":"kv_log_store_buffer_unconsumed_epoch_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Buffer State","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":287,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_rewind_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","query":"sum(rate(kv_log_store_rewind_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Rewind Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":288,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(kv_log_store_rewind_delay_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, executor_id, connector, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","query":"histogram_quantile(1.0, sum(rate(kv_log_store_rewind_delay_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, executor_id, connector, sink_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Rewind delay (second)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total size of chunks buffered in a barrier","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":289,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_sink_chunk_buffer_size{job=~\"$job\",instance=~\"$node\"}) by (sink_id, actor_id) * on(actor_id) group_left(sink_name) sink_info{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}} - actor {{actor_id}}","metric":"","query":"sum(stream_sink_chunk_buffer_size{job=~\"$job\",instance=~\"$node\"}) by (sink_id, actor_id) * on(actor_id) group_left(sink_name) sink_info{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Chunk Buffer Size","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Sink Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":26},"height":null,"hideTimeOverride":false,"id":290,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Kafka high watermark by source and partition and source latest message by partition, source and actor","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":291,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_kafka_high_watermark{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"high watermark: source={{source_id}} partition={{partition}}","metric":"","query":"source_kafka_high_watermark{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_latest_message_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"latest msg: source={{source_id}} partition={{partition}} actor_id={{actor_id}}","metric":"","query":"source_latest_message_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kafka high watermark and source latest message","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Current number of messages in producer queues","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":292,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_msg_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Count in Producer Queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Current total size of messages in producer queues","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":293,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_msg_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_msg_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Size in Producer Queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of messages transmitted (produced) to Kafka brokers","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":294,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_tx_msgs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_tx_msgs{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Produced Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of messages consumed, not including ignored messages (due to offset, etc), from Kafka brokers.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":295,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_rx_msgs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_rx_msgs{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Received Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages awaiting transmission to broker","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":296,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_outbuf_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_outbuf_msg_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Count Pending to Transmit (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages in-flight to broker awaiting response","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":297,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_waitresp_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_waitresp_msg_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inflight Message Count (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of transmission errors","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":298,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_tx_errs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_tx_errs{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Error Count When Transmitting (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of receive errors","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":299,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rx_errs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_rx_errs{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Error Count When Receiving (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of requests timed out","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":300,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_req_timeouts{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_req_timeouts{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Timeout Request Count (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Broker latency / round-trip time in milli seconds","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":301,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_avg{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_avg{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p75{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_p75{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p90{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_p90{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_p99{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"RTT (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Broker throttling time in milliseconds","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":302,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_avg{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_avg{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p75{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_p75{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p90{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_p90{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_p99{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Throttle Time (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Age of metadata from broker for this topic (milliseconds)","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":303,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_metadata_age{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}","metric":"","query":"rdkafka_topic_metadata_age{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Topic Metadata_age Age","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Batch sizes in bytes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":304,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_avg{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_avg{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p75{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_p75{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p90{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_p90{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_p99{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p99_99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_p99_99{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_out_of_range{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_out_of_range{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Batch message counts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_avg{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_avg{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p75{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_p75{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p90{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_p90{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_p99{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p99_99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_p99_99{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_out_of_range{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_out_of_range{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Topic Batch Messages","transformations":[],"transparent":false,"type":"timeseries"}],"timeFrom":null,"timeShift":null,"title":"Topic Batch Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages ready to be produced in transmit queue","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":305,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_xmit_msgq_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","query":"rdkafka_topic_partition_xmit_msgq_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message to be Transmitted","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of pre-fetched messages in fetch queue","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":306,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_fetchq_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","query":"rdkafka_topic_partition_fetchq_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message in pre fetch queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Next offset to fetch","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":307,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_next_offset{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","query":"rdkafka_topic_partition_next_offset{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Next offset to fetch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Last committed offset","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":308,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_committed_offset{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","query":"rdkafka_topic_partition_committed_offset{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Committed Offset","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Kafka Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":27},"height":null,"hideTimeOverride":false,"id":309,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":310,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} read @ {{instance}}","metric":"","query":"sum(rate(connection_read_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} write @ {{instance}}","metric":"","query":"sum(rate(connection_write_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Network throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":311,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} read @ {{instance}}","metric":"","query":"sum(rate(connection_read_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} write @ {{instance}}","metric":"","query":"sum(rate(connection_write_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"S3 throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":312,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} read @ {{instance}}","metric":"","query":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} write @ {{instance}}","metric":"","query":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} total read @ {{instance}}","metric":"","query":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} total write @ {{instance}}","metric":"","query":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"gRPC throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":313,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_io_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 {{op_type}} err[{{error_kind}}] @ {{instance}}","metric":"","query":"sum(irate(connection_io_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_io_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} grpc {{op_type}} err[{{error_kind}}] @ {{instance}}","metric":"","query":"sum(rate(connection_io_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_io_err_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} total {{op_type}} err[{{error_kind}}] @ {{instance}}","metric":"","query":"sum(rate(connection_io_err_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"IO error rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":314,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(connection_count{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 @ {{instance}}","metric":"","query":"sum(connection_count{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(connection_count{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}) by (job, instance, connection_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} @ {{instance}}","metric":"","query":"sum(connection_count{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}) by (job, instance, connection_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Existing connection count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":315,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_create_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 @ {{instance}}","metric":"","query":"sum(irate(connection_create_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_create_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} @ {{instance}}","metric":"","query":"sum(irate(connection_create_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create new connection rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":316,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 @ {{instance}}","metric":"","query":"sum(irate(connection_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} @ {{instance}}","metric":"","query":"sum(irate(connection_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create new connection err rate","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Network connection","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":28},"height":null,"hideTimeOverride":false,"id":317,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"iceberg write qps","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":318,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"iceberg_write_qps{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} @ {{sink_id}}","metric":"","query":"iceberg_write_qps{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Qps Of Iceberg Writer","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":319,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 @ {{sink_id}}","metric":"","query":"histogram_quantile(0.5, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 @ {{sink_id}}","metric":"","query":"histogram_quantile(0.99, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax @ {{sink_id}}","metric":"","query":"histogram_quantile(1.0, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, sink_id)(rate(iceberg_write_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(iceberg_write_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg @ {{sink_id}}","metric":"","query":"sum by(le, sink_id)(rate(iceberg_write_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(iceberg_write_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Latency Of Iceberg Writer","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":320,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"iceberg_rolling_unfushed_data_file{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} @ {{sink_id}}","metric":"","query":"iceberg_rolling_unfushed_data_file{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iceberg rolling unfushed data file","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":321,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"iceberg_position_delete_cache_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} @ {{sink_id}}","metric":"","query":"iceberg_position_delete_cache_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iceberg position delete cache num","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":322,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"iceberg_partition_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} @ {{sink_id}}","metric":"","query":"iceberg_partition_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iceberg partition num","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Iceberg Sink Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":29},"height":null,"hideTimeOverride":false,"id":323,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":324,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_success_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_success_count - {{instance}}","metric":"","query":"sum(rate(udf_success_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_failure_count - {{instance}}","metric":"","query":"sum(rate(udf_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_retry_count - {{instance}}","metric":"","query":"sum(rate(udf_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_success_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_success_count - {{link}} {{name}} {{fragment_id}}","metric":"","query":"sum(rate(udf_success_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_failure_count - {{link}} {{name}} {{fragment_id}}","metric":"","query":"sum(rate(udf_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_retry_count - {{instance}}","metric":"","query":"sum(rate(udf_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Calls Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":325,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(udf_input_chunk_rows_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / sum(irate(udf_input_chunk_rows_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_input_chunk_rows_avg - {{link}} {{name}} {{fragment_id}}","metric":"","query":"sum(irate(udf_input_chunk_rows_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / sum(irate(udf_input_chunk_rows_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Input Chunk Rows","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":326,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.50, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.50, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(udf_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / sum(irate(udf_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_avg - {{instance}}","metric":"","query":"sum(irate(udf_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / sum(irate(udf_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, link, name, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_p99_by_name - {{link}} {{name}} {{fragment_id}}","metric":"","query":"histogram_quantile(0.99, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, link, name, fragment_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(udf_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / sum(irate(udf_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_avg_by_name - {{link}} {{name}} {{fragment_id}}","metric":"","query":"sum(irate(udf_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / sum(irate(udf_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":327,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_input_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_throughput_rows - {{instance}}","metric":"","query":"sum(rate(udf_input_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_input_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_throughput_rows - {{link}} {{name}} {{fragment_id}}","metric":"","query":"sum(rate(udf_input_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Throughput (rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":328,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_throughput_bytes - {{instance}}","metric":"","query":"sum(rate(udf_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_throughput_bytes - {{link}} {{name}} {{fragment_id}}","metric":"","query":"sum(rate(udf_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / (1024*1024)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Throughput (bytes)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Currently only embedded JS UDF supports this. Others will always show 0.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":329,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(udf_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_memory_usage - {{instance}}","metric":"","query":"sum(udf_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(udf_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_memory_usage - {{name}} {{fragment_id}}","metric":"","query":"sum(udf_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Memory Usage (bytes)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"User Defined Function","transformations":[],"transparent":false,"type":"row"}],"refresh":"","rows":[],"schemaVersion":12,"sharedCrosshair":true,"style":"dark","tags":["risingwave"],"templating":{"list":[{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, instance)","description":"Reporting instance of the metric","hide":0,"includeAll":true,"label":"Node","multi":true,"name":"node","options":[],"query":{"query":"label_values(process_cpu_seconds_total, instance)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, job)","description":"Reporting job of the metric","hide":0,"includeAll":true,"label":"Job","multi":true,"name":"job","options":[],"query":{"query":"label_values(process_cpu_seconds_total, job)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(table_info, table_id)","description":"Reporting table id of the metric","hide":0,"includeAll":true,"label":"Table","multi":true,"name":"table","options":[],"query":{"query":"label_values(table_info, table_id)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"}]},"time":{"from":"now-30m","to":"now"},"timepicker":{"hidden":false,"nowDelay":null,"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone":"browser","title":"risingwave_dev_dashboard","uid":"Ecy3uV1nz","version":0}
+{"__inputs":[],"annotations":{"list":[]},"description":"RisingWave Dev Dashboard","editable":true,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"panels":[{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":1,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Information about actors","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]}},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":2,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true},"repeat":null,"repeatDirection":null,"span":6,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"group(actor_info{job=~\"$job\",instance=~\"$node\"}) by (actor_id, fragment_id, compute_node)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"group(actor_info{job=~\"$job\",instance=~\"$node\"}) by (actor_id, fragment_id, compute_node)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true},"indexByName":{"actor_id":0,"compute_node":2,"fragment_id":1}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Information about state tables. Column `materialized_view_id` is the id of the materialized view that this state table belongs to.","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]}},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":3,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true},"repeat":null,"repeatDirection":null,"span":6,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name, table_type, materialized_view_id, fragment_id, compaction_group_id)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name, table_type, materialized_view_id, fragment_id, compaction_group_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"State Table Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true},"indexByName":{"compaction_group_id":5,"fragment_id":4,"materialized_view_id":3,"table_id":0,"table_name":1,"table_type":2}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Actor count per compute node","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]}},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":4,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true},"repeat":null,"repeatDirection":null,"span":6,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"count(actor_info{job=~\"$job\",instance=~\"$node\"}) by (compute_node)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"count(actor_info{job=~\"$job\",instance=~\"$node\"}) by (compute_node)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Count (Group By Compute Node)","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true},"indexByName":{"compaction_group_id":5,"fragment_id":4,"materialized_view_id":3,"table_id":0,"table_name":1,"table_type":2}}}],"transparent":false,"type":"table"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Actor/Table Id Info","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":1},"height":null,"hideTimeOverride":false,"id":5,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of each type of RisingWave components alive.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":6,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_type}}","metric":"","query":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The memory usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":7,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Memory","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Memory usage relative to k8s resource limit of container. Only works in K8s environment","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":8,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(avg by(namespace, pod) (container_memory_working_set_bytes{namespace=~\"$namespace\",pod=~\"$pod\",container=~\"$component\"})) / (  sum by(namespace, pod) (kube_pod_container_resource_limits{namespace=~\"$namespace\", pod=~\"$pod\", container=\"$component\", resource=\"memory\", unit=\"byte\"}))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg memory usage @ {{job}} @ {{instance}}","metric":"","query":"(avg by(namespace, pod) (container_memory_working_set_bytes{namespace=~\"$namespace\",pod=~\"$pod\",container=~\"$component\"})) / (  sum by(namespace, pod) (kube_pod_container_resource_limits{namespace=~\"$namespace\", pod=~\"$pod\", container=\"$component\", resource=\"memory\", unit=\"byte\"}))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Memory relative","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The CPU usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":9,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cpu usage (total) - {{job}} @ {{instance}}","metric":"","query":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (job, instance) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cpu usage (avg per core) - {{job}} @ {{instance}}","metric":"","query":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (job, instance) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"CPU usage relative to k8s resource limit of container. Only works in K8s environment","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":10,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\",container=~\"$component\",pod=~\"$pod\"}[$__rate_interval])) by (namespace, pod)) / (sum(kube_pod_container_resource_limits{namespace=~\"$namespace\",pod=~\"$pod\",container=~\"$component\", resource=\"cpu\"}) by (namespace, pod))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cpu usage @ {{job}} @ {{instance}}","metric":"","query":"(sum(rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\",container=~\"$component\",pod=~\"$pod\"}[$__rate_interval])) by (namespace, pod)) / (sum(kube_pod_container_resource_limits{namespace=~\"$namespace\",pod=~\"$pod\",container=~\"$component\", resource=\"cpu\"}) by (namespace, pod))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU relative","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"RW cluster can configure multiple meta nodes to achieve high availability. One is the leader and the rest are the followers.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":11,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(meta_num{job=~\"$job\",instance=~\"$node\"}) by (worker_addr,role)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_addr}} @ {{role}}","metric":"","query":"sum(meta_num{job=~\"$job\",instance=~\"$node\"}) by (worker_addr,role)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Meta Cluster","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Cluster Node","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":2},"height":null,"hideTimeOverride":false,"id":12,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The rate of successful recovery attempts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":13,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Recovery Successful Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of failed reocovery attempts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":14,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Failed recovery attempts","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Time spent in a successful recovery attempt","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":15,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency pmax - {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by (le) (rate(recovery_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by (le) (rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency avg","metric":"","query":"sum by (le) (rate(recovery_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by (le) (rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Recovery latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Recovery","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":3},"height":null,"hideTimeOverride":false,"id":16,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of barriers that have been ingested but not completely processed. This metric reflects the current level of congestion within the system.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":17,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all_barrier","metric":"","query":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"in_flight_barrier_nums{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"in_flight_barrier","metric":"","query":"in_flight_barrier_nums{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The time that the data between two consecutive barriers gets fully processed, i.e. the computation results are made durable into materialized views or sink to external systems. This metric shows to users the freshness of materialized views.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":18,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_avg","metric":"","query":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The duration from the last committed barrier's epoch time to the current time. This metric reflects the data freshness of the system. During this time, no new data has been committed.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":19,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"timestamp(last_committed_barrier_time{job=~\"$job\",instance=~\"$node\"}) - last_committed_barrier_time{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_pending_time","metric":"","query":"timestamp(last_committed_barrier_time{job=~\"$job\",instance=~\"$node\"}) - last_committed_barrier_time{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier pending time (secs)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":20,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","query":"sum(rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Each query is executed in parallel with a user-defined parallelism. This figure shows the throughput of each parallelism. The throughput of all the parallelism added up is equal to Source Throughput(rows).","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":21,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(source_partition_input_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor={{actor_id}} source={{source_id}} partition={{partition}} fragment_id={{fragment_id}}","metric":"","query":"rate(source_partition_input_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s) Per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of bytes read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":22,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum by (source_id, source_name, fragment_id)(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","query":"(sum by (source_id, source_name, fragment_id)(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Each query is executed in parallel with a user-defined parallelism. This figure shows the throughput of each parallelism. The throughput of all the parallelism added up is equal to Source Throughput(MB/s).","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":23,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor={{actor_id}} source={{source_id}} partition={{partition}} fragment_id={{fragment_id}}","metric":"","query":"(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))/(1000*1000)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s) Per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":24,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_source_backfill_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","query":"sum(rate(stream_source_backfill_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Backfill Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Monitor each source upstream, 0 means the upstream is not normal, 1 means the source is ready.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":25,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_status_is_up{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source_id={{source_id}}, source_name={{source_name}} @ {{instance}}","metric":"","query":"source_status_is_up{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Upstream Status","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Source Split Change Events frequency by source_id and actor_id","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":26,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_source_split_change_event_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_name}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_source_split_change_event_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Split Change Events frequency(events/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Kafka Consumer Lag Size by source_id, partition and actor_id","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":27,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_min(source_kafka_high_watermark{job=~\"$job\",instance=~\"$node\"} - on(source_id, partition) group_right() source_latest_message_id{job=~\"$job\",instance=~\"$node\"}, 0)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_id}} partition={{partition}} actor_id={{actor_id}}","metric":"","query":"clamp_min(source_kafka_high_watermark{job=~\"$job\",instance=~\"$node\"} - on(source_id, partition) group_right() source_latest_message_id{job=~\"$job\",instance=~\"$node\"}, 0)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kafka Consumer Lag Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of rows streamed into each sink per second. For sinks with 'sink_decouple = true', please refer to the 'Sink Metrics' section","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":28,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id) * on(sink_id) group_left(sink_name) group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}}","metric":"","query":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id) * on(sink_id) group_left(sink_name) group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (sink_id, sink_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Throughput(rows/s) *","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of rows streamed into each sink per second. For sinks with 'sink_decouple = true', please refer to the 'Sink Metrics' section","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":29,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id, actor_id) * on(actor_id) group_left(sink_name) sink_info{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}} - actor {{actor_id}}","metric":"","query":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id, actor_id) * on(actor_id) group_left(sink_name) sink_info{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Throughput(rows/s) per Partition *","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized view per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":30,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id) * on(table_id) group_left(table_name) group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mview {{table_id}} {{table_name}}","metric":"","query":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id) * on(table_id) group_left(table_name) group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized view per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":31,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) * on(fragment_id, table_id) group_left(table_name) table_info{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mview {{table_id}} {{table_name}} - actor {{actor_id}} fragment_id {{fragment_id}}","metric":"","query":"rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) * on(fragment_id, table_id) group_left(table_name) table_info{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s) per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been read from the backfill snapshot","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":32,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_snapshot_read_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_backfill_snapshot_read_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Snapshot Read Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been output from the backfill upstream","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":33,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_upstream_output_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_backfill_upstream_output_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Upstream Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The duration between the time point when the scheduled barrier needs to be sent and the time point when the barrier gets actually sent to all the compute nodes. Developers can thus detect any internal congestion.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":34,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_send_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_send_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_avg","metric":"","query":"rate(meta_barrier_send_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_send_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Send Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":35,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"max(sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_avg","metric":"","query":"max(sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier In-Flight Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":36,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p999 - {{instance}}","metric":"","query":"histogram_quantile(0.999, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_pmax - {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_avg - {{instance}}","metric":"","query":"sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Sync Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":37,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_wait_commit_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_wait_commit_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_avg","metric":"","query":"rate(meta_barrier_wait_commit_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_wait_commit_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Wait Commit Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of actors that have processed the earliest in-flight barriers per second. This metric helps users to detect potential congestion or stuck in the system.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":38,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_barrier_manager_progress{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"rate(stream_barrier_manager_progress{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Earliest In-Flight Barrier Progress","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":4},"height":null,"hideTimeOverride":false,"id":39,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been read from the cdc backfill snapshot","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":40,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_cdc_backfill_snapshot_read_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_cdc_backfill_snapshot_read_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CDC Backfill Snapshot Read Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been output from the cdc backfill upstream","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":41,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_cdc_backfill_upstream_output_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_cdc_backfill_upstream_output_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CDC Backfill Upstream Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":42,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lag p50 - {{table_name}}","metric":"","query":"histogram_quantile(0.5, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lag p99 - {{table_name}}","metric":"","query":"histogram_quantile(0.99, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lag pmax - {{table_name}}","metric":"","query":"histogram_quantile(1.0, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CDC Consume Lag Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":43,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(cdc_source_error{job=~\"$job\",instance=~\"$node\"}) by (connector_name, source_id, error_msg)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{connector_name}}: {{error_msg}} ({{source_id}})","metric":"","query":"sum(cdc_source_error{job=~\"$job\",instance=~\"$node\"}) by (connector_name, source_id, error_msg)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CDC Source Errors","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming CDC","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":5},"height":null,"hideTimeOverride":false,"id":44,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"We first record the total blocking duration(ns) of output buffer of each actor. It shows how much time it takes an actor to process a message, i.e. a barrier, a watermark or rows of data, on average. Then we divide this duration by 1 second and show it as a percentage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":45,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, downstream_fragment_id) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}->{{downstream_fragment_id}}","metric":"","query":"avg(rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, downstream_fragment_id) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Output Blocking Time Ratio (Backpressure)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":46,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_actor_input_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, upstream_fragment_id) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}<-{{upstream_fragment_id}}","metric":"","query":"avg(rate(stream_actor_input_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, upstream_fragment_id) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Input Blocking Time Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":47,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_actor_in_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, upstream_fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}<-{{upstream_fragment_id}}","metric":"","query":"sum(rate(stream_actor_in_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, upstream_fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_in_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}}","metric":"","query":"rate(stream_actor_in_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Input Throughput (rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":48,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_actor_out_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_actor_out_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_out_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}}","metric":"","query":"rate(stream_actor_out_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Output Throughput (rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The operator-level memory usage statistics collected by each LRU cache","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":49,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (table_id, desc)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table {{table_id}} desc: {{desc}}","metric":"","query":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (table_id, desc)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_memory_usage{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table {{table_id}} actor {{actor_id}} desc: {{desc}}","metric":"","query":"stream_memory_usage{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Memory Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Memory usage aggregated by materialized views","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":50,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id) group_left(materialized_view_id) table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized view {{materialized_view_id}}","metric":"","query":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id) group_left(materialized_view_id) table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Memory Usage of Materialized Views","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":51,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"temporal join cache miss, table_id {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"temporal join cache miss, table_id {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Temporal Join Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":52,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_materialize_cache_hit_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache hit count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_materialize_cache_hit_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_materialize_cache_total_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total cached count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_materialize_cache_total_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_hit_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache hit count - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_materialize_cache_hit_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_total_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total cached count - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_materialize_cache_total_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialize Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":53,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache lookup count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_over_window_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_cache_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache miss count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_over_window_cache_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_over_window_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache lookup count - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_over_window_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_over_window_cache_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache miss count - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_over_window_cache_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_range_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"partition range cache lookup count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_over_window_range_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_range_cache_left_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"partition range cache left miss count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_over_window_range_cache_left_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_range_cache_right_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"partition range cache right miss count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_over_window_range_cache_right_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Over Window Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":54,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, fragment_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Join executor cache miss ratio - - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} fragment {{fragment_id}}","metric":"","query":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, fragment_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n appendonly cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream lookup cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream temporal join cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Materialize executor cache miss ratio - table {{table_id}} fragment {{fragment_id}}  {{instance}}","metric":"","query":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_over_window_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Over window cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_over_window_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_over_window_range_cache_left_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_range_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Over window partition range cache left miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_over_window_range_cache_left_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_range_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_over_window_range_cache_right_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_range_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Over window partition range cache right miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_over_window_range_cache_right_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_range_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Miss Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":55,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_barrier_align_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000) by (fragment_id,wait_side, executor)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}} {{wait_side}} {{executor}}","metric":"","query":"avg(rate(stream_barrier_align_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000) by (fragment_id,wait_side, executor)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_barrier_align_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}} fragment {{fragment_id}} {{wait_side}} {{executor}}","metric":"","query":"rate(stream_barrier_align_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Barrier Align Per Second","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":56,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - fragment {{fragment_id}} - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - fragment {{fragment_id}} - {{job}}","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p999 - fragment {{fragment_id}} - {{job}}","metric":"","query":"histogram_quantile(0.999, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - fragment {{fragment_id}} - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, fragment_id, job)(rate(stream_merge_barrier_align_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le,fragment_id,job) (rate(stream_merge_barrier_align_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - fragment {{fragment_id}} - {{job}}","metric":"","query":"sum by(le, fragment_id, job)(rate(stream_merge_barrier_align_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le,fragment_id,job) (rate(stream_merge_barrier_align_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Merger Barrier Align","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":57,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_join_actor_input_waiting_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000) by (fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}","metric":"","query":"avg(rate(stream_join_actor_input_waiting_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000) by (fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_actor_input_waiting_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}}","metric":"","query":"rate(stream_join_actor_input_waiting_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Actor Input Blocking Time Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":58,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_join_match_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000) by (fragment_id,side)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}} {{side}}","metric":"","query":"avg(rate(stream_join_match_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000) by (fragment_id,side)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_match_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}} {{side}}","metric":"","query":"rate(stream_join_match_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Actor Match Duration Per Second","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Multiple rows with distinct primary keys may have the same join key. This metric counts the number of join keys in the executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":59,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (fragment_id, side)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}} {{side}}","metric":"","query":"sum(stream_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (fragment_id, side)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}} {{side}}","metric":"","query":"stream_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of matched rows on the opposite side","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":60,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - fragment {{fragment_id}} table_id {{table_id}} - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - fragment {{fragment_id}} table_id {{table_id}} - {{job}}","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - fragment {{fragment_id}} table_id {{table_id}} - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, fragment_id, table_id) (rate(stream_join_matched_join_keys_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, fragment_id, table_id) (rate(stream_join_matched_join_keys_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - fragment {{fragment_id}} table_id {{table_id}} - {{job}}","metric":"","query":"sum by(le, job, fragment_id, table_id) (rate(stream_join_matched_join_keys_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, fragment_id, table_id) (rate(stream_join_matched_join_keys_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) >= 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Executor Matched Rows","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":61,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_agg_chunk_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level cache miss - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_agg_chunk_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_agg_chunk_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level total lookups - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_agg_chunk_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_chunk_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level cache miss  - table {{table_id}} actor {{actor_id}}}","metric":"","query":"rate(stream_agg_chunk_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_chunk_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level total lookups  - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_chunk_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Executor Cache Statistics For Each StreamChunk","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in each hash aggregation executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":62,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_agg_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg cached keys count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_agg_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_agg_distinct_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg distinct cached keys count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_agg_distinct_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg cached keys count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_agg_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_distinct_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg distinct cached keys count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_agg_distinct_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of dirty (unflushed) groups in each hash aggregation executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":63,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_agg_dirty_groups_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_agg_dirty_groups_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_dirty_groups_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_agg_dirty_groups_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Dirty Groups Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The total heap size of dirty (unflushed) groups in each hash aggregation executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":64,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_agg_dirty_groups_heap_size{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups heap size | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_agg_dirty_groups_heap_size{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_dirty_groups_heap_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups heap size | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_agg_dirty_groups_heap_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Dirty Groups Heap Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in each top_n executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":65,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_group_top_n_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_group_top_n_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_group_top_n_appendonly_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n appendonly cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_group_top_n_appendonly_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_group_top_n_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_group_top_n_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_group_top_n_appendonly_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n appendonly cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_group_top_n_appendonly_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"TopN Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in temporal join executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":66,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_temporal_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal Join cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_temporal_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_temporal_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal Join cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_temporal_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Temporal Join Cache Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in lookup executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":88},"height":null,"hideTimeOverride":false,"id":67,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_lookup_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lookup cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_lookup_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_lookup_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lookup cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_lookup_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lookup Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in over window executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":88},"height":null,"hideTimeOverride":false,"id":68,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_over_window_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"over window cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_over_window_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_over_window_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"over window cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_over_window_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_over_window_range_cache_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"over window partition range cache entry count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_over_window_range_cache_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Over Window Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"When enabled, this metric shows the input throughput of each executor.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":96},"height":null,"hideTimeOverride":false,"id":69,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_executor_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_identity, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_identity}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_executor_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_identity, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_executor_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_identity}} actor {{actor_id}}","metric":"","query":"rate(stream_executor_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The actor-level memory usage statistics reported by TaskLocalAlloc. (Disabled by default)","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":96},"height":null,"hideTimeOverride":false,"id":70,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(actor_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}","metric":"","query":"sum(actor_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"actor_memory_usage{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}}","metric":"","query":"actor_memory_usage{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Memory Usage (TaskLocalAlloc)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Actors","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":6},"height":null,"hideTimeOverride":false,"id":71,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":72,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_actor_execution_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_actor_execution_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Execution Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":73,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":74,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":8},"height":null,"hideTimeOverride":false,"id":75,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":8},"height":null,"hideTimeOverride":false,"id":76,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":77,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":16},"height":null,"hideTimeOverride":false,"id":78,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":16},"height":null,"hideTimeOverride":false,"id":79,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":80,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":24},"height":null,"hideTimeOverride":false,"id":81,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":24},"height":null,"hideTimeOverride":false,"id":82,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":83,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":32},"height":null,"hideTimeOverride":false,"id":84,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":32},"height":null,"hideTimeOverride":false,"id":85,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":86,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":40},"height":null,"hideTimeOverride":false,"id":87,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Avg Time","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Actors (Tokio)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":7},"height":null,"hideTimeOverride":false,"id":88,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":89,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{up_fragment_id}}->{{down_fragment_id}}","metric":"","query":"rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fragment-level Remote Exchange Send Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":90,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{up_fragment_id}}->{{down_fragment_id}}","metric":"","query":"rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fragment-level Remote Exchange Recv Throughput","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Exchange","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":91,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Errors that happened during computation. Check the logs for detailed error message.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":92,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_compute_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, executor_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{executor_name}} (fragment_id={{fragment_id}})","metric":"","query":"sum(user_compute_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, executor_name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compute Errors by Type","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Errors that happened during source data ingestion. Check the logs for detailed error message.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":93,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_source_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{source_name}} (source_id={{source_id}} fragment_id={{fragment_id}})","metric":"","query":"sum(user_source_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, source_id, source_name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Errors by Type","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Errors that happened during data sink out. Check the logs for detailed error message.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":94,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_sink_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, sink_id, sink_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{sink_name}} (sink_id={{sink_id}} fragment_id={{fragment_id}})","metric":"","query":"sum(user_sink_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, sink_id, sink_name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Errors by Type","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"User Streaming Errors","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":9},"height":null,"hideTimeOverride":false,"id":95,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"row"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":96,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{query_id}} : {{source_stage_id}}.{{source_task_id}} -> {{target_stage_id}}.{{target_task_id}}","metric":"","query":"batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Exchange Recv Row Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":97,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_task_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"batch_task_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Mpp Task Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"All memory usage of batch executors in bytes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":98,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"compute_batch_total_mem{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"compute_batch_total_mem{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"frontend_batch_total_mem{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"frontend_batch_total_mem{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Mem Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":99,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_heartbeat_worker_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"batch_heartbeat_worker_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Heartbeat Worker Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":100,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Row SeqScan Next Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Disk throughputs of spilling-out in the bacth query engine","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":101,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(batch_spill_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}} @ {{instance}}","metric":"","query":"sum(rate(batch_spill_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(batch_spill_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}} @ {{instance}}","metric":"","query":"sum(rate(batch_spill_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Spill Throughput","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Batch Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":10},"height":null,"hideTimeOverride":false,"id":102,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":103,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{table_id}} @ {{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_sst_store_block_request_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total_meta_miss_count - {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Hummock has three parts of memory usage: 1. Meta Cache 2. Block CacheThis metric shows the real memory usage of each of these three caches.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":104,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta cache - {{job}} @ {{instance}}","metric":"","query":"avg(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"data cache - {{job}} @ {{instance}}","metric":"","query":"avg(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(state_store_prefetch_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"prefetch cache - {{job}} @ {{instance}}","metric":"","query":"avg(state_store_prefetch_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":105,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='meta_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta cache miss ratio - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"(sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='meta_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_sst_store_block_request_counts{type='data_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='data_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache miss ratio - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"(sum(rate(state_store_sst_store_block_request_counts{type='data_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='data_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) >= 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Miss Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of the estimated hit ratio of a block while in the block cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":106,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_max(histogram_quantile(0.1, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache efficienfy - p10 - {{job}} @ {{instance}}","metric":"","query":"clamp_max(histogram_quantile(0.1, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_max(histogram_quantile(0.25, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache efficienfy - p25 - {{job}} @ {{instance}}","metric":"","query":"clamp_max(histogram_quantile(0.25, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_max(histogram_quantile(0.5, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache efficienfy - p50 - {{job}} @ {{instance}}","metric":"","query":"clamp_max(histogram_quantile(0.5, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_max(histogram_quantile(0.75, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache efficienfy - p75 - {{job}} @ {{instance}}","metric":"","query":"clamp_max(histogram_quantile(0.75, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_max(histogram_quantile(0.9, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache efficienfy - p90 - {{job}} @ {{instance}}","metric":"","query":"clamp_max(histogram_quantile(0.9, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_max(histogram_quantile(1.0, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache efficienfy - p100 - {{job}} @ {{instance}}","metric":"","query":"clamp_max(histogram_quantile(1.0, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Block Cache Efficiency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":107,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_scan_key_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type, table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter keys flow - {{table_id}} @ {{type}} @ {{instance}}","metric":"","query":"sum(rate(state_store_iter_scan_key_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type, table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iter keys flow","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":108,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts p50 - {{table_id}} @ {{job}} @ {{type}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts p99 - {{table_id}} @ {{job}} @ {{type}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts pmax - {{table_id}} @ {{job}} @ {{type}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts avg  - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Merged SSTs","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of the latency of Get operations that have been issued to the state store.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":109,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_get_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, table_id)(rate(state_store_get_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Duration - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of the time spent on iterator initialization.Histogram of the time spent on iterator scanning.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":110,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time p50 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time p99 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time pmax - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_iter_init_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, iter_type) (rate(state_store_iter_init_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time avg - {{iter_type}} {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(state_store_iter_init_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, iter_type) (rate(state_store_iter_init_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time p50 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time p99 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time pmax - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_iter_scan_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, iter_type) (rate(state_store_iter_scan_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time avg - {{iter_type}} {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(state_store_iter_scan_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, iter_type) (rate(state_store_iter_scan_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Duration - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":111,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter false positive count  - {{table_id}} - {{type}}","metric":"","query":"sum(irate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter positive count - {{table_id}} - {{type}}","metric":"","query":"sum(irate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter check count- {{table_id}} - {{type}}","metric":"","query":"sum(irate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom Filter Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Positive / Total","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":112,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter positive rate - {{table_id}} - {{type}}","metric":"","query":"(sum(rate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)) >= 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom Filter Positive Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"False-Positive / Total","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":113,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(((sum(rate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read req bloom filter false positive rate - {{table_id}} - {{type}}","metric":"","query":"(((sum(rate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))) >= 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom Filter False-Positive Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":114,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_iter_slow_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"state_store_iter_slow_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Slow Fetch Meta Unhits","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":115,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_shared_buffer_hit_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"shared_buffer hit - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_get_shared_buffer_hit_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id, iter_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{iter_type}} - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_iter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id, iter_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":116,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Size - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":117,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Size - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":118,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read p50 - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read p99 - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read pmax - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Read Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":119,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_iter_in_progress_counts{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Existing {{iter_type}} count @ {{table_id}}","metric":"","query":"state_store_iter_in_progress_counts{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_log_op_type_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, op_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter_log op count @ {{table_id}} {{op_type}}","metric":"","query":"sum(rate(state_store_iter_log_op_type_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, op_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Count - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size of a single key-value pair when reading by operation Get.Operation Get gets a single key-value pair with respect to a caller-specified key. If the key does not exist in the storage, the size of key is counted into this metric and the size of value is 0.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":120,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance) + sum(rate(state_store_get_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_get_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance) + sum(rate(state_store_get_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Throughput - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size of all the key-value paris when reading by operation Iter.Operation Iter scans a range of key-value pairs.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":121,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_iter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Throughput - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":122,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration avg - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fetch Meta Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":123,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_iter_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"state_store_iter_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fetch Meta Unhits","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock (Read)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":11},"height":null,"hideTimeOverride":false,"id":124,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric shows the real memory usage of uploader.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":125,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploading memory - {{job}} @ {{instance}}","metric":"","query":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_uploading_task_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploading task size - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_uploader_uploading_task_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_imm_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploader imm size - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_uploader_imm_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_imm_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance) - sum(state_store_uploader_uploading_task_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unflushed imm size - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_uploader_imm_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance) - sum(state_store_uploader_uploading_task_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance) - sum(state_store_uploader_imm_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"orphan imm size - {{job}} @ {{instance}}","metric":"","query":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance) - sum(state_store_uploader_imm_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_old_value_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"old value size - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_old_value_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader Memory Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of time spent on compacting shared buffer to remote storage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":126,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 Sync duration - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 Sync duration - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax Sync duration - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_sync_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg Sync duration - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance) (rate(state_store_sync_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_uploader_upload_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 upload task duration - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_uploader_upload_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_uploader_upload_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 upload task duration - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_uploader_upload_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_uploader_upload_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax upload task duration - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_uploader_upload_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Build and Sync Sstable Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":127,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.5, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write p50 - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(0.5, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.99, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write p99 - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(0.99, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write pmax - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Write Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":128,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_merge_imm_task_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"merge imm tasks - {{table_id}} @ {{instance}}","metric":"","query":"sum(irate(state_store_merge_imm_task_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_spill_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Uploader spill tasks - {{uploader_stage}} @ {{instance}}","metric":"","query":"sum(irate(state_store_spill_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_uploading_task_count{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploading task count - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_uploader_uploading_task_count{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_syncing_epoch_count{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"syncing epoch count - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_uploader_syncing_epoch_count{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader - Tasks Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":129,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_merge_imm_memory_sz{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Merging tasks memory size - {{table_id}} @ {{instance}}","metric":"","query":"sum(rate(state_store_merge_imm_memory_sz{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_spill_task_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Uploading tasks size - {{uploader_stage}} @ {{instance}}","metric":"","query":"sum(rate(state_store_spill_task_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader - Task Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":130,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write batch - {{table_id}} @ {{job}} @ {{instance}} ","metric":"","query":"sum(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"l0 - {{job}} @ {{instance}} ","metric":"","query":"sum(rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":131,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer avg - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":132,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_write_batch_tuple_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write_batch_kv_pair_count - {{table_id}} @ {{instance}}","metric":"","query":"sum(irate(state_store_write_batch_tuple_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Item Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":133,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_write_batch_size_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id) / sum(rate(state_store_write_batch_size_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"shared_buffer - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_write_batch_size_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id) / sum(rate(state_store_write_batch_size_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_shared_buffer_to_sstable_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance) / sum(rate(compactor_shared_buffer_to_sstable_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sync - {{job}} @ {{instance}}","metric":"","query":"sum(rate(compactor_shared_buffer_to_sstable_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance) / sum(rate(compactor_shared_buffer_to_sstable_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric shows the statistics of mem_table size on flush. By default only max (p100) is shown.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":134,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_write_batch_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, table_id, job, instance) (rate(state_store_write_batch_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance) (rate(state_store_write_batch_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, table_id, job, instance) (rate(state_store_write_batch_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Batch Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":135,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_mem_table_spill_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mem table spill table id - {{table_id}} @ {{instance}}","metric":"","query":"sum(irate(state_store_mem_table_spill_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Mem Table Spill Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":136,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Checkpoint Sync Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":137,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_event_handler_pending_event{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"sum(state_store_event_handler_pending_event{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Event handler pending event number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":138,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_event_handler_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, event_type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 {{event_type}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_event_handler_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, event_type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_event_handler_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, event_type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 {{event_type}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_event_handler_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, event_type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_event_handler_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, event_type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax {{event_type}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_event_handler_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, event_type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_uploader_wait_poll_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 finished_task_wait_poll {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_uploader_wait_poll_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_uploader_wait_poll_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 finished_task_wait_poll {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_uploader_wait_poll_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_uploader_wait_poll_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax finished_task_wait_poll {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_uploader_wait_poll_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Event handle latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock (Write)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":12},"height":null,"hideTimeOverride":false,"id":139,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of SSTables at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":140,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","query":"sum(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"SSTable Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size(KB) of SSTables at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":141,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","query":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"SSTable Size(KB)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The  of bytes that have been written by commit epoch per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":142,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_commit_write_throughput{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{table_id}}","metric":"","query":"sum(rate(storage_commit_write_throughput{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Commit Flush Bytes by Table","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have completed or failed","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":143,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_frequency{result!='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task_type}} - {{result}} - group-{{group}} @ {{compactor}}","metric":"","query":"sum(storage_level_compact_frequency{result!='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Failure Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have completed or failed","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":144,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_frequency{result='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task_type}} - {{result}} - group-{{group}} @ {{compactor}}","metric":"","query":"sum(storage_level_compact_frequency{result='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Success Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have been skipped.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":145,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_skip_compact_frequency{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (level, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{level}}-{{type}}","metric":"","query":"sum(rate(storage_skip_compact_frequency{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (level, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Skip Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Avg l0 select_level_count of the compact task, and categorize it according to different cg, levels and task types","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":146,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, group, type)(irate(storage_l0_compact_level_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, group, type)(irate(storage_l0_compact_level_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg cg{{group}}@{{type}}","metric":"","query":"sum by(le, group, type)(irate(storage_l0_compact_level_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, group, type)(irate(storage_l0_compact_level_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task L0 Select Level Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Avg file count of the compact task, and categorize it according to different cg, levels and task types","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":147,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, group, type)(irate(storage_compact_task_file_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, group, type)(irate(storage_compact_task_file_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg cg{{group}}@{{type}}","metric":"","query":"sum by(le, group, type)(irate(storage_compact_task_file_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, group, type)(irate(storage_compact_task_file_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task File Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The distribution of the compact task size triggered, including p90 and max. and categorize it according to different cg, levels and task types.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":148,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - cg{{group}}@{{type}}","metric":"","query":"histogram_quantile(0.9, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - cg{{group}}@{{type}}","metric":"","query":"histogram_quantile(1.0, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task Size Distribution","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that are running.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":149,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(storage_compact_task_pending_num{job=~\"$job\",instance=~\"$node\"}) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor_task_count - {{job}} @ {{instance}}","metric":"","query":"avg(storage_compact_task_pending_num{job=~\"$job\",instance=~\"$node\"}) by(job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(storage_compact_task_pending_parallelism{job=~\"$job\",instance=~\"$node\"}) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor_task_pending_parallelism - {{job}} @ {{instance}}","metric":"","query":"avg(storage_compact_task_pending_parallelism{job=~\"$job\",instance=~\"$node\"}) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compactor Running Task Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"compact-task: The total time have been spent on compaction.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":150,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task p50 - {{job}}","metric":"","query":"histogram_quantile(0.5, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task p90 - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task pmax - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range p90 - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range pmax - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get-table-id p90 - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get-table-id pmax - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io p90 - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io pmax - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(compute_refill_cache_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compute_apply_version_duration_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(compute_refill_cache_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le)(rate(compactor_compact_task_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(compactor_compact_task_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task avg","metric":"","query":"sum by(le)(rate(compactor_compact_task_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(compactor_compact_task_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le)(rate(state_store_compact_sst_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(state_store_compact_sst_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range avg","metric":"","query":"sum by(le)(rate(state_store_compact_sst_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(state_store_compact_sst_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"KBs read from next level during history compactions to next level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":151,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job) + sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}}","metric":"","query":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job) + sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}}","metric":"","query":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"flush - {{job}}","metric":"","query":"sum(rate(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_fast_compact_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fast compact - {{job}}","metric":"","query":"sum(rate(compactor_fast_compact_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of bytes that have been written by compaction.Flush refers to the process of compacting Memtables to SSTables at Level 0.Write refers to the process of compacting SSTables at one level to another level.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":152,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}}","metric":"","query":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"flush - {{job}}","metric":"","query":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Write Bytes(GiB)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Write amplification is the amount of bytes written to the remote storage by compaction for each one byte of flushed SSTable data. Write amplification is by definition higher than 1.0 because we write each piece of data to L0, and then write it again to an SSTable, and then compaction may read this piece of data and write it to a new SSTable, that's another write.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":153,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) / sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write amplification","metric":"","query":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) / sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Write Amplification","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of SSTables that is being compacted at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":154,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_level_compact_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","query":"storage_level_compact_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compacting SSTable Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"num of compact_task","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":155,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_level_compact_task_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task}}","metric":"","query":"storage_level_compact_task_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compacting Task Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":156,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from next level","metric":"","query":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from current level","metric":"","query":"sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} write to next level","metric":"","query":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"KBs Read/Write by Level","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":157,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_write_sstn{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} write to next level","metric":"","query":"sum(irate(storage_level_compact_write_sstn{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_read_sstn_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from next level","metric":"","query":"sum(irate(storage_level_compact_read_sstn_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_read_sstn_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from current level","metric":"","query":"sum(irate(storage_level_compact_read_sstn_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Count of SSTs Read/Write by level","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"For observing bloom_filter size, sstable file size, sstable block size etc.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":158,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.50, sum(rate(compactor_sstable_bloom_filter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom_filter_size_p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.50, sum(rate(compactor_sstable_bloom_filter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(rate(compactor_sstable_bloom_filter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom_filter_size_p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(rate(compactor_sstable_bloom_filter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Bloom Filter Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"For observing sstable file size","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":159,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.50, sum(rate(compactor_sstable_file_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sstable_file_size_p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.50, sum(rate(compactor_sstable_file_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(rate(compactor_sstable_file_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sstable_file_size_p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(rate(compactor_sstable_file_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable File Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"For observing sstable block size","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":160,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.50, sum(rate(compactor_sstable_block_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sstable_block_size_p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.50, sum(rate(compactor_sstable_block_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(rate(compactor_sstable_block_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sstable_block_size_p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(rate(compactor_sstable_block_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Block Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"For observing avg key and value count","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":161,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_key_size - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_value_size - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Avg Key And Value Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total time of operations which read from remote storage when enable prefetch","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":88},"height":null,"hideTimeOverride":false,"id":162,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io p90 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Remote Read Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":88},"height":null,"hideTimeOverride":false,"id":163,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_iter_scan_key_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter keys flow - {{type}} @ {{instance}} ","metric":"","query":"sum(rate(compactor_iter_scan_key_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compactor Iter keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"bytes of Lsm tree needed to reach balance","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":96},"height":null,"hideTimeOverride":false,"id":164,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_compact_pending_bytes{job=~\"$job\",instance=~\"$node\"}) by (instance, group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact pending bytes - {{group}} @ {{instance}} ","metric":"","query":"sum(storage_compact_pending_bytes{job=~\"$job\",instance=~\"$node\"}) by (instance, group)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lsm Compact Pending Bytes","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"compression ratio of each level of the lsm tree","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":96},"height":null,"hideTimeOverride":false,"id":165,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_compact_level_compression_ratio{job=~\"$job\",instance=~\"$node\"}) by (instance, group, level, algorithm)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lsm compression ratio - cg{{group}} @ L{{level}} - {{algorithm}} {{instance}}","metric":"","query":"sum(storage_compact_level_compression_ratio{job=~\"$job\",instance=~\"$node\"}) by (instance, group, level, algorithm)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lsm Level Compression Ratio","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Compaction","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":13},"height":null,"hideTimeOverride":false,"id":166,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":167,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":168,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, type, job, instance)(rate(object_store_operation_latency_sum{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(object_store_operation_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, type, job, instance)(rate(object_store_operation_latency_sum{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(object_store_operation_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":169,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type!~'streaming_upload_write_bytes|streaming_read_read_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_operation_latency_count{type!~'streaming_upload_write_bytes|streaming_read_read_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type=~'upload|delete',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{media_type}}-write - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_operation_latency_count{type=~'upload|delete',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type=~'read|readv|list|metadata',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{media_type}}-read - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_operation_latency_count{type=~'read|readv|list|metadata',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":170,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":171,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Failure Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":172,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_request_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_request_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Retry Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"There are two types of operations: 1. GET, SELECT, and DELETE, they cost 0.0004 USD per 1000 requests. 2. PUT, COPY, POST, LIST, they cost 0.005 USD per 1000 requests.Reading from S3 across different regions impose extra cost. This metric assumes 0.01 USD per 1GB data transfer. Please checkout AWS's pricing model for more accurate calculation.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"$"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":173,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}) * 0.01 / 1000 / 1000 / 1000","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"(Cross Region) Data Transfer Cost","metric":"","query":"sum(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}) * 0.01 / 1000 / 1000 / 1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_operation_latency_count{type=~'read|streaming_read_start|streaming_read_init',job=~\"$job\",instance=~\"$node\"}) * 0.0004 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GET, SELECT, and all other Requests Cost","metric":"","query":"sum(object_store_operation_latency_count{type=~'read|streaming_read_start|streaming_read_init',job=~\"$job\",instance=~\"$node\"}) * 0.0004 / 1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_operation_latency_count{type=~'upload|streaming_upload|streaming_upload_start|s3_upload_part|streaming_upload_finish|list',job=~\"$job\",instance=~\"$node\"}) * 0.005 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"PUT, COPY, POST, LIST Requests Cost","metric":"","query":"sum(object_store_operation_latency_count{type=~'upload|streaming_upload|streaming_upload_start|s3_upload_part|streaming_upload_finish|list',job=~\"$job\",instance=~\"$node\"}) * 0.005 / 1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Estimated S3 Cost (Realtime)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric uses the total size of data in S3 at this second to derive the cost of storing data for a whole month. The price is 0.023 USD per GB. Please checkout AWS's pricing model for more accurate calculation.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"$"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":174,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance) * 0.023 / 1000 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Monthly Storage Cost","metric":"","query":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance) * 0.023 / 1000 / 1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Estimated S3 Cost (Monthly)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Object Storage","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":14},"height":null,"hideTimeOverride":false,"id":175,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":176,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_hybrid_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - hybrid - {{op}} @ {{instance}}","metric":"","query":"sum(rate(foyer_hybrid_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hybrid Cache Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":177,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{name}} - hybrid - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{name}} - hybrid - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{name}} - hybrid - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{name}} - hybrid - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hybrid Cache Op Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":178,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_hybrid_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) / (sum(rate(foyer_hybrid_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) + sum(rate(foyer_hybrid_op_total{op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - hybrid - hit ratio @ {{instance}}","metric":"","query":"sum(rate(foyer_hybrid_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) / (sum(rate(foyer_hybrid_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) + sum(rate(foyer_hybrid_op_total{op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hybrid Cache Hit Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":179,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_memory_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - memory - {{op}} @ {{instance}}","metric":"","query":"sum(rate(foyer_memory_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Cache Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":180,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(foyer_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (name, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - memory - size @ {{instance}}","metric":"","query":"sum(foyer_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (name, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Cache Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":181,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_memory_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) / (sum(rate(foyer_memory_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) + sum(rate(foyer_memory_op_total{op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - memory - hit ratio @ {{instance}}","metric":"","query":"sum(rate(foyer_memory_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) / (sum(rate(foyer_memory_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) + sum(rate(foyer_memory_op_total{op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Cache Hit Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":182,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"sum(rate(foyer_storage_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":183,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_inner_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"sum(rate(foyer_storage_inner_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache Inner Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":184,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache Op Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":185,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache Inner Op Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":186,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) / (sum(rate(foyer_storage_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) + sum(rate(foyer_storage_op_total{op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - storage - hit ratio @ {{instance}}","metric":"","query":"sum(rate(foyer_storage_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) / (sum(rate(foyer_storage_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) + sum(rate(foyer_storage_op_total{op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache Hit Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":187,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(foyer_storage_region{job=~\"$job\",instance=~\"$node\"}) by (name, type, instance) * on(name, instance) group_left() avg(foyer_storage_region_size_bytes{job=~\"$job\",instance=~\"$node\"}) by (name, type, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - {{type}} region - size @ {{instance}}","metric":"","query":"sum(foyer_storage_region{job=~\"$job\",instance=~\"$node\"}) by (name, type, instance) * on(name, instance) group_left() avg(foyer_storage_region_size_bytes{job=~\"$job\",instance=~\"$node\"}) by (name, type, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Region Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":188,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_disk_io_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - disk - {{op}} @ {{instance}}","metric":"","query":"sum(rate(foyer_storage_disk_io_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Disk Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":189,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{name}} - disk - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{name}} - disk - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{name}} - disk - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{name}} - disk - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Disk Op Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":190,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_disk_io_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - disk - {{op}} @ {{instance}}","metric":"","query":"sum(rate(foyer_storage_disk_io_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Disk Op Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":191,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (type, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} file cache refill - {{op}} @ {{instance}}","metric":"","query":"sum(rate(refill_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (type, op, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=~\"meta|data\",op!~\"filtered|ignored\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (type, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} file cache refill - {{op}} @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=~\"meta|data\",op!~\"filtered|ignored\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (type, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":192,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} file cache - {{op}} @ {{instance}}","metric":"","query":"sum(rate(refill_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Data Refill Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":193,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{foyer}} cache refill - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{foyer}} cache refill - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{foyer}} cache refill - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{foyer}} cache refill - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":194,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(refill_queue_total) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"refill queue length @ {{instance}}","metric":"","query":"sum(refill_queue_total) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill Queue Length","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":195,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"parent_meta\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"parent meta lookup {{op}} @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=\"parent_meta\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inheritance - Parent Meta Lookup Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":196,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"parent_meta\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / (sum(rate(refill_total{type=\"parent_meta\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) + sum(rate(refill_total{type=\"parent_meta\",op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"parent meta lookup hit ratio @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=\"parent_meta\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / (sum(rate(refill_total{type=\"parent_meta\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) + sum(rate(refill_total{type=\"parent_meta\",op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)) >= 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inheritance - Parent Meta Lookup Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":197,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"unit_inheritance\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unit inheritance {{op}} @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=\"unit_inheritance\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inheritance - Unit inheritance Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":88},"height":null,"hideTimeOverride":false,"id":198,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"unit_inheritance\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / (sum(rate(refill_total{type=\"unit_inheritance\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) + sum(rate(refill_total{type=\"unit_inheritance\",op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unit inheritance ratio @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=\"unit_inheritance\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / (sum(rate(refill_total{type=\"unit_inheritance\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) + sum(rate(refill_total{type=\"unit_inheritance\",op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)) >= 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inheritance - Unit inheritance Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":88},"height":null,"hideTimeOverride":false,"id":199,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"block\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block refill {{op}} @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=\"block\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Block Refill Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":96},"height":null,"hideTimeOverride":false,"id":200,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"block\",op=\"success\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / sum(rate(refill_total{type=\"block\",op=\"unfiltered\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block refill ratio @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=\"block\",op=\"success\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / sum(rate(refill_total{type=\"block\",op=\"unfiltered\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) >= 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Block Refill Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Item numbers of the recent filter.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":96},"height":null,"hideTimeOverride":false,"id":201,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(recent_filter_items{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"items @ {{instance}}","metric":"","query":"sum(recent_filter_items{job=~\"$job\",instance=~\"$node\"}) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Recent Filter Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":104},"height":null,"hideTimeOverride":false,"id":202,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(recent_filter_ops{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recent filter {{op}} @ {{instance}}","metric":"","query":"sum(rate(recent_filter_ops{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Recent Filter Ops","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock Tiered Cache","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":15},"height":null,"hideTimeOverride":false,"id":203,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":204,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time p50 - {{method}} @ {{lock_type}} @ {{lock_name}}","metric":"","query":"histogram_quantile(0.5, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method, lock_name, lock_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time p99 - {{method}} @ {{lock_type}} @ {{lock_name}}","metric":"","query":"histogram_quantile(0.99, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method, lock_name, lock_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time pmax - {{method}} @ {{lock_type}} @ {{lock_name}}","metric":"","query":"histogram_quantile(1.0, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method, lock_name, lock_type))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lock Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":205,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time p50 - {{method}}","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time p99 - {{method}}","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time pmax - {{method}}","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Real Process Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":206,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version size","metric":"","query":"storage_version_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":207,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"current version id","metric":"","query":"storage_current_version_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"checkpoint version id","metric":"","query":"storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min pinned version id","metric":"","query":"storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_safepoint_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min safepoint version id","metric":"","query":"storage_min_safepoint_version_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Id","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":208,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"max committed epoch","metric":"","query":"storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_safe_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"safe epoch","metric":"","query":"storage_safe_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min pinned epoch","metric":"","query":"storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Epoch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":209,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_key_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","query":"storage_version_stats{metric='total_key_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_value_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","query":"storage_version_stats{metric='total_value_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Table Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":210,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_materialized_view_stats{metric='materialized_view_total_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{metric}}, mv id - {{table_id}} ","metric":"","query":"storage_materialized_view_stats{metric='materialized_view_total_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":211,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_key_count',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","query":"storage_version_stats{metric='total_key_count',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Table KV Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"\nObjects are classified into 3 groups:\n- not referenced by versions: these object are being deleted from object store.\n- referenced by non-current versions: these objects are stale (not in the latest version), but those old versions may still be in use (e.g. long-running pinning). Thus those objects cannot be deleted at the moment.\n- referenced by current version: these objects are in the latest version.\n\nAdditionally, a metric on all objects (including dangling ones) is updated with low-frequency. The metric is updated right before full GC. So subsequent full GC may reduce the actual value significantly, without updating the metric.\n                    ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":212,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","query":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","query":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","query":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_total_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all objects (including dangling ones)","metric":"","query":"storage_total_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Refer to `Object Total Number` panel for classification of objects.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":213,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","query":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","query":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","query":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_total_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all objects, including dangling ones","metric":"","query":"storage_total_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"total number of hummock version delta log","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":214,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_delta_log_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"delta log total number","metric":"","query":"storage_delta_log_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Delta Log Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"hummock version checkpoint latency","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":215,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(storage_version_checkpoint_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(storage_version_checkpoint_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_avg","metric":"","query":"rate(storage_version_checkpoint_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(storage_version_checkpoint_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Checkpoint Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"When certain per compaction group threshold is exceeded (e.g. number of level 0 sub-level in LSMtree), write op to that compaction group is stopped temporarily. Check log for detail reason of write stop.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":216,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_write_stop_compaction_groups{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compaction_group_{{compaction_group_id}}","metric":"","query":"storage_write_stop_compaction_groups{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Stop Compaction Groups","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"total number of attempts to trigger full GC","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":217,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_full_gc_trigger_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"full_gc_trigger_count","metric":"","query":"storage_full_gc_trigger_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Full GC Trigger Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"the object id watermark used in last full GC","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":218,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_full_gc_last_object_id_watermark{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"full_gc_last_object_id_watermark","metric":"","query":"storage_full_gc_last_object_id_watermark{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Full GC Last Watermark","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":219,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Event Loop Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The times of move_state_table occurs","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":220,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_move_state_table_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"move table cg{{group}}","metric":"","query":"sum(storage_move_state_table_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}) by (group)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Move State Table Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of state_tables in each CG","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":221,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_state_table_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"state table cg{{group}}","metric":"","query":"sum(irate(storage_state_table_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"State Table Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of branched_sst in each CG","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":222,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_branched_sst_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"branched sst cg{{group}}","metric":"","query":"sum(irate(storage_branched_sst_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Branched SST Count","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":223,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total backup job count since the Meta node starts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":224,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"backup_job_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"job count","metric":"","query":"backup_job_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Job Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Latency of backup jobs since the Meta node starts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":225,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time p50 - {{state}}","metric":"","query":"histogram_quantile(0.5, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time p99 - {{state}}","metric":"","query":"histogram_quantile(0.99, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time pmax - {{state}}","metric":"","query":"histogram_quantile(1.0, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Job Process Time","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Backup Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":17},"height":null,"hideTimeOverride":false,"id":226,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":227,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":228,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Drop latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":229,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"GetCatalog latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Catalog Service","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":18},"height":null,"hideTimeOverride":false,"id":230,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":231,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"AddWorkerNode latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":232,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"ListAllNodes latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Cluster Service","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":19},"height":null,"hideTimeOverride":false,"id":233,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":234,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CreateMaterializedView latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":235,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"DropMaterializedView latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":236,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Flush latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Stream Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":20},"height":null,"hideTimeOverride":false,"id":237,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":238,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UnpinVersionBefore latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":239,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UnpinSnapshotBefore latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":240,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"ReportCompactionTasks latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":241,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"GetNewSstIds latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Hummock Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":21},"height":null,"hideTimeOverride":false,"id":242,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":243,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_report_compaction_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_counts - {{instance}}","metric":"","query":"sum(irate(state_store_report_compaction_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"compaction_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":244,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_version_before_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_version_before_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_avg","metric":"","query":"sum(irate(state_store_unpin_version_before_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_version_before_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"version_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":245,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latencyp90 - {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_pin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_pin_snapshot_latency_count[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_avg","metric":"","query":"sum(irate(state_store_pin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_pin_snapshot_latency_count[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_snapshot_latency_count[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_avg","metric":"","query":"sum(irate(state_store_unpin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_snapshot_latency_count[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_unpin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(irate(state_store_unpin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"snapshot_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":246,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_pin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_counts - {{instance}}","metric":"","query":"sum(irate(state_store_pin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_counts - {{instance}}","metric":"","query":"sum(irate(state_store_unpin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"snapshot_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":247,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_get_new_sst_ids_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_get_new_sst_ids_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_avg","metric":"","query":"sum(irate(state_store_get_new_sst_ids_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_get_new_sst_ids_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"table_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":248,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_get_new_sst_ids_latency_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_counts - {{instance}}","metric":"","query":"sum(irate(state_store_get_new_sst_ids_latency_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"table_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":249,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_report_compaction_task_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_report_compaction_task_latency_count[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_avg","metric":"","query":"sum(irate(state_store_report_compaction_task_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_report_compaction_task_latency_count[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"compaction_latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC: Hummock Meta Client","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":22},"height":null,"hideTimeOverride":false,"id":250,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of active sessions","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":251,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Active Sessions","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":252,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Per Second (Local Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":253,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Per Second (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":254,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of running query in distributed execution mode","metric":"","query":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Running Queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":255,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of rejected query in distributed execution mode","metric":"","query":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Rejected queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":256,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of completed query in distributed execution mode","metric":"","query":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Completed Queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":257,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":258,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency (Local Query Mode)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Frontend","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":23},"height":null,"hideTimeOverride":false,"id":259,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":260,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(lru_runtime_loop_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"rate(lru_runtime_loop_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager loop count per sec","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":261,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_eviction_policy{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"lru_eviction_policy{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager eviction policy","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":262,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_latest_sequence{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"lru_latest_sequence{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_watermark_sequence{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"lru_watermark_sequence{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager sequence","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":263,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jemalloc_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The allocated memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":264,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_active_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jemalloc_active_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The active memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":265,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_resident_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jemalloc_resident_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The resident memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":266,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_metadata_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jemalloc_metadata_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The metadata memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":267,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jvm_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jvm_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The allocated memory of jvm","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":268,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jvm_active_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jvm_active_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The active memory of jvm","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":269,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_current_watermark_time_ms{job=~\"$job\",instance=~\"$node\"} - on() group_right() lru_evicted_watermark_time_ms{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table {{table_id}} actor {{actor_id}} desc: {{desc}}","metric":"","query":"lru_current_watermark_time_ms{job=~\"$job\",instance=~\"$node\"} - on() group_right() lru_evicted_watermark_time_ms{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager diff between current watermark and evicted watermark time (ms) for actors","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Memory manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":270,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":271,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(source_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_type}} @ {{source_id}}","metric":"","query":"rate(source_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Source Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":272,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(connector_sink_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector_type}} @ {{sink_id}}","metric":"","query":"rate(connector_sink_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Sink Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Connector Node","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":25},"height":null,"hideTimeOverride":false,"id":273,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":274,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id, sink_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 @ {{sink_id}} {{sink_name}} ({{connector}})","metric":"","query":"histogram_quantile(0.5, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id, sink_name))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id, sink_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 @ {{sink_id}} {{sink_name}} ({{connector}})","metric":"","query":"histogram_quantile(0.99, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id, sink_name))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id, sink_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax @ {{sink_id}} {{sink_name}} ({{connector}})","metric":"","query":"histogram_quantile(1.0, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id, sink_name))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, type, job, instance, sink_id, sink_name)(rate(sink_commit_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance, sink_id, sink_name) (rate(sink_commit_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg @ {{sink_id}} {{sink_name}} ({{connector}})","metric":"","query":"sum by(le, type, job, instance, sink_id, sink_name)(rate(sink_commit_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance, sink_id, sink_name) (rate(sink_commit_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Commit Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":275,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"log_store_latest_write_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"latest write epoch @ {{sink_id}} {{sink_name}} ({{connector}}) actor {{actor_id}}","metric":"","query":"log_store_latest_write_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"latest read epoch @ {{sink_id}} {{sink_name}} ({{connector}}) actor {{actor_id}}","metric":"","query":"log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"kv_log_store_buffer_unconsumed_min_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Kv log store uncomsuned min epoch @ {{sink_id}} {{sink_name}} ({{connector}}) actor {{actor_id}}","metric":"","query":"kv_log_store_buffer_unconsumed_min_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Read/Write Epoch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":276,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(max(log_store_latest_write_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, actor_id, sink_name)- max(log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, actor_id, sink_name)) / (2^16) / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Consume lag @ {{sink_id}} {{sink_name}} ({{connector}}) actor {{actor_id}}","metric":"","query":"(max(log_store_latest_write_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, actor_id, sink_name)- max(log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, actor_id, sink_name)) / (2^16) / 1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Lag","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":277,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(log_store_reader_wait_new_future_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id, actor_id, sink_name) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Backpressure @ {{sink_id}} {{sink_name}} ({{connector}}) actor {{actor_id}}","metric":"","query":"avg(rate(log_store_reader_wait_new_future_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id, actor_id, sink_name) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Backpressure Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":278,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_min((max(log_store_first_write_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, actor_id, sink_name)- max(log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, actor_id, sink_name)) / (2^16) / 1000, 0)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Consume persistent log lag @ {{sink_id}} {{sink_name}} ({{connector}}) actor {{actor_id}}","metric":"","query":"clamp_min((max(log_store_first_write_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, actor_id, sink_name)- max(log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, actor_id, sink_name)) / (2^16) / 1000, 0)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Consume Persistent Log Lag","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":279,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_read_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} ({{connector}})","metric":"","query":"sum(rate(log_store_read_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id, sink_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Consume Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":280,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_read_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, connector, sink_id, actor_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} ({{connector}}) actor {{actor_id}} @ {{instance}}","metric":"","query":"sum(rate(log_store_read_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, connector, sink_id, actor_id, sink_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Log Store Consume Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":281,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_write_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{sink_id}} {{sink_name}} ({{connector}})","metric":"","query":"sum(rate(log_store_write_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id, sink_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Write Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":282,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_write_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, connector, sink_id, actor_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} ({{connector}}) actor {{actor_id}} {{instance}}","metric":"","query":"sum(rate(log_store_write_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, connector, sink_id, actor_id, sink_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Log Store Write Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":283,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_read_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (actor_id, connector, sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} actor {{actor_id}} ({{connector}})","metric":"","query":"sum(rate(kv_log_store_storage_read_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (actor_id, connector, sink_id, sink_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Read Storage Row Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":284,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_read_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (actor_id, connector, sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} actor {{actor_id}} ({{connector}})","metric":"","query":"sum(rate(kv_log_store_storage_read_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (actor_id, connector, sink_id, sink_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Read Storage Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":285,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_write_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (actor_id, connector, sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} actor {{actor_id}} ({{connector}})","metric":"","query":"sum(rate(kv_log_store_storage_write_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (actor_id, connector, sink_id, sink_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Write Storage Row Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":286,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_write_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (actor_id, connector, sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} actor {{actor_id}} ({{connector}})","metric":"","query":"sum(rate(kv_log_store_storage_write_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (actor_id, connector, sink_id, sink_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Write Storage Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":287,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"kv_log_store_buffer_unconsumed_item_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Unconsumed item count @ {{sink_id}} {{sink_name}} ({{connector}}) actor {{actor_id}}","metric":"","query":"kv_log_store_buffer_unconsumed_item_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"kv_log_store_buffer_unconsumed_row_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Unconsumed row count @ {{sink_id}} {{sink_name}} ({{connector}}) actor {{actor_id}}","metric":"","query":"kv_log_store_buffer_unconsumed_row_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"kv_log_store_buffer_unconsumed_epoch_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Unconsumed epoch count @ {{sink_id}} {{sink_name}} ({{connector}}) actor {{actor_id}}","metric":"","query":"kv_log_store_buffer_unconsumed_epoch_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Buffer State","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":288,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_rewind_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (actor_id, connector, sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} actor {{actor_id}} ({{connector}})","metric":"","query":"sum(rate(kv_log_store_rewind_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (actor_id, connector, sink_id, sink_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Rewind Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":289,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(kv_log_store_rewind_delay_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, actor_id, connector, sink_id, sink_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} actor {{actor_id}} ({{connector}})","metric":"","query":"histogram_quantile(1.0, sum(rate(kv_log_store_rewind_delay_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, actor_id, connector, sink_id, sink_name))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Rewind delay (second)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total size of chunks buffered in a barrier","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":290,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_sink_chunk_buffer_size{job=~\"$job\",instance=~\"$node\"}) by (sink_id, actor_id, sink_name) * on(actor_id) group_left(sink_name) sink_info{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}} - actor {{actor_id}}","metric":"","query":"sum(stream_sink_chunk_buffer_size{job=~\"$job\",instance=~\"$node\"}) by (sink_id, actor_id, sink_name) * on(actor_id) group_left(sink_name) sink_info{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Chunk Buffer Size","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Sink Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":26},"height":null,"hideTimeOverride":false,"id":291,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Kafka high watermark by source and partition and source latest message by partition, source and actor","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":292,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_kafka_high_watermark{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"high watermark: source={{source_id}} partition={{partition}}","metric":"","query":"source_kafka_high_watermark{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_latest_message_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"latest msg: source={{source_id}} partition={{partition}} actor_id={{actor_id}}","metric":"","query":"source_latest_message_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kafka high watermark and source latest message","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Current number of messages in producer queues","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":293,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_msg_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Count in Producer Queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Current total size of messages in producer queues","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":294,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_msg_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_msg_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Size in Producer Queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of messages transmitted (produced) to Kafka brokers","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":295,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_tx_msgs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_tx_msgs{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Produced Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of messages consumed, not including ignored messages (due to offset, etc), from Kafka brokers.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":296,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_rx_msgs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_rx_msgs{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Received Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages awaiting transmission to broker","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":297,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_outbuf_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_outbuf_msg_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Count Pending to Transmit (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages in-flight to broker awaiting response","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":298,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_waitresp_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_waitresp_msg_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inflight Message Count (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of transmission errors","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":299,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_tx_errs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_tx_errs{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Error Count When Transmitting (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of receive errors","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":300,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rx_errs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_rx_errs{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Error Count When Receiving (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of requests timed out","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":301,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_req_timeouts{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_req_timeouts{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Timeout Request Count (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Broker latency / round-trip time in milli seconds","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":302,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_avg{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_avg{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p75{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_p75{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p90{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_p90{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_p99{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"RTT (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Broker throttling time in milliseconds","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":303,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_avg{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_avg{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p75{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_p75{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p90{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_p90{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_p99{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Throttle Time (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Age of metadata from broker for this topic (milliseconds)","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":304,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_metadata_age{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}","metric":"","query":"rdkafka_topic_metadata_age{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Topic Metadata_age Age","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Batch sizes in bytes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":305,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_avg{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_avg{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p75{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_p75{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p90{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_p90{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_p99{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p99_99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_p99_99{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_out_of_range{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_out_of_range{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Batch message counts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_avg{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_avg{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p75{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_p75{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p90{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_p90{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_p99{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p99_99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_p99_99{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_out_of_range{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_out_of_range{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Topic Batch Messages","transformations":[],"transparent":false,"type":"timeseries"}],"timeFrom":null,"timeShift":null,"title":"Topic Batch Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages ready to be produced in transmit queue","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":306,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_xmit_msgq_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","query":"rdkafka_topic_partition_xmit_msgq_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message to be Transmitted","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of pre-fetched messages in fetch queue","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":307,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_fetchq_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","query":"rdkafka_topic_partition_fetchq_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message in pre fetch queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Next offset to fetch","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":308,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_next_offset{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","query":"rdkafka_topic_partition_next_offset{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Next offset to fetch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Last committed offset","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":309,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_committed_offset{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","query":"rdkafka_topic_partition_committed_offset{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Committed Offset","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Kafka Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":27},"height":null,"hideTimeOverride":false,"id":310,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":311,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} read @ {{instance}}","metric":"","query":"sum(rate(connection_read_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} write @ {{instance}}","metric":"","query":"sum(rate(connection_write_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Network throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":312,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} read @ {{instance}}","metric":"","query":"sum(rate(connection_read_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} write @ {{instance}}","metric":"","query":"sum(rate(connection_write_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"S3 throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":313,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} read @ {{instance}}","metric":"","query":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} write @ {{instance}}","metric":"","query":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} total read @ {{instance}}","metric":"","query":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} total write @ {{instance}}","metric":"","query":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"gRPC throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":314,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_io_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 {{op_type}} err[{{error_kind}}] @ {{instance}}","metric":"","query":"sum(irate(connection_io_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_io_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} grpc {{op_type}} err[{{error_kind}}] @ {{instance}}","metric":"","query":"sum(rate(connection_io_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_io_err_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} total {{op_type}} err[{{error_kind}}] @ {{instance}}","metric":"","query":"sum(rate(connection_io_err_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"IO error rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":315,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(connection_count{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 @ {{instance}}","metric":"","query":"sum(connection_count{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(connection_count{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}) by (job, instance, connection_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} @ {{instance}}","metric":"","query":"sum(connection_count{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}) by (job, instance, connection_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Existing connection count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":316,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_create_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 @ {{instance}}","metric":"","query":"sum(irate(connection_create_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_create_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} @ {{instance}}","metric":"","query":"sum(irate(connection_create_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create new connection rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":317,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 @ {{instance}}","metric":"","query":"sum(irate(connection_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} @ {{instance}}","metric":"","query":"sum(irate(connection_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create new connection err rate","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Network connection","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":28},"height":null,"hideTimeOverride":false,"id":318,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"iceberg write qps","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":319,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"iceberg_write_qps{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} actor {{actor_id}}","metric":"","query":"iceberg_write_qps{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Qps Of Iceberg Writer","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":320,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id, sink_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 @ {{sink_id}} {{sink_name}}","metric":"","query":"histogram_quantile(0.5, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id, sink_name))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id, sink_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 @ {{sink_id}} {{sink_name}}","metric":"","query":"histogram_quantile(0.99, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id, sink_name))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id, sink_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax @ {{sink_id}} {{sink_name}}","metric":"","query":"histogram_quantile(1.0, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id, sink_name))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, type, job, instance, sink_id, sink_name)(rate(iceberg_write_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance, sink_id, sink_name) (rate(iceberg_write_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg @ {{sink_id}} {{sink_name}}","metric":"","query":"sum by(le, type, job, instance, sink_id, sink_name)(rate(iceberg_write_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance, sink_id, sink_name) (rate(iceberg_write_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Latency Of Iceberg Writer","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":321,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"iceberg_rolling_unflushed_data_file{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} actor {{actor_id}}","metric":"","query":"iceberg_rolling_unflushed_data_file{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iceberg rolling unfushed data file","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":322,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"iceberg_position_delete_cache_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} actor {{actor_id}}","metric":"","query":"iceberg_position_delete_cache_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iceberg position delete cache num","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":323,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"iceberg_partition_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} actor {{actor_id}}","metric":"","query":"iceberg_partition_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iceberg partition num","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Iceberg Sink Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":29},"height":null,"hideTimeOverride":false,"id":324,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":325,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_success_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_success_count - {{instance}}","metric":"","query":"sum(rate(udf_success_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_failure_count - {{instance}}","metric":"","query":"sum(rate(udf_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_retry_count - {{instance}}","metric":"","query":"sum(rate(udf_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_success_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_success_count - {{link}} {{name}} {{fragment_id}}","metric":"","query":"sum(rate(udf_success_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_failure_count - {{link}} {{name}} {{fragment_id}}","metric":"","query":"sum(rate(udf_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_retry_count - {{instance}}","metric":"","query":"sum(rate(udf_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Calls Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":326,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(udf_input_chunk_rows_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / sum(irate(udf_input_chunk_rows_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_input_chunk_rows_avg - {{link}} {{name}} {{fragment_id}}","metric":"","query":"sum(irate(udf_input_chunk_rows_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / sum(irate(udf_input_chunk_rows_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Input Chunk Rows","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":327,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.50, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.50, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(udf_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / sum(irate(udf_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_avg - {{instance}}","metric":"","query":"sum(irate(udf_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / sum(irate(udf_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, link, name, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_p99_by_name - {{link}} {{name}} {{fragment_id}}","metric":"","query":"histogram_quantile(0.99, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, link, name, fragment_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(udf_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / sum(irate(udf_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_avg_by_name - {{link}} {{name}} {{fragment_id}}","metric":"","query":"sum(irate(udf_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / sum(irate(udf_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":328,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_input_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_throughput_rows - {{instance}}","metric":"","query":"sum(rate(udf_input_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_input_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_throughput_rows - {{link}} {{name}} {{fragment_id}}","metric":"","query":"sum(rate(udf_input_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Throughput (rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":329,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_throughput_bytes - {{instance}}","metric":"","query":"sum(rate(udf_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_throughput_bytes - {{link}} {{name}} {{fragment_id}}","metric":"","query":"sum(rate(udf_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / (1024*1024)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Throughput (bytes)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Currently only embedded JS UDF supports this. Others will always show 0.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":330,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(udf_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_memory_usage - {{instance}}","metric":"","query":"sum(udf_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(udf_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_memory_usage - {{name}} {{fragment_id}}","metric":"","query":"sum(udf_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Memory Usage (bytes)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"User Defined Function","transformations":[],"transparent":false,"type":"row"}],"refresh":"","rows":[],"schemaVersion":12,"sharedCrosshair":true,"style":"dark","tags":["risingwave"],"templating":{"list":[{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, instance)","description":"Reporting instance of the metric","hide":0,"includeAll":true,"label":"Node","multi":true,"name":"node","options":[],"query":{"query":"label_values(process_cpu_seconds_total, instance)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, job)","description":"Reporting job of the metric","hide":0,"includeAll":true,"label":"Job","multi":true,"name":"job","options":[],"query":{"query":"label_values(process_cpu_seconds_total, job)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(table_info, table_id)","description":"Reporting table id of the metric","hide":0,"includeAll":true,"label":"Table","multi":true,"name":"table","options":[],"query":{"query":"label_values(table_info, table_id)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"}]},"time":{"from":"now-30m","to":"now"},"timepicker":{"hidden":false,"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone":"browser","title":"risingwave_dev_dashboard","uid":"Ecy3uV1nz","version":0}
diff --git a/docker/dashboards/risingwave-user-dashboard.json b/docker/dashboards/risingwave-user-dashboard.json
index 037679704e38f..0d0619aad29ec 100644
--- a/docker/dashboards/risingwave-user-dashboard.json
+++ b/docker/dashboards/risingwave-user-dashboard.json
@@ -1 +1 @@
-{"__inputs":[],"annotations":{"list":[]},"description":"RisingWave Dashboard","editable":true,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"panels":[{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":1,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Information about actors","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":2,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true,"sortBy":[]},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"group(actor_info{job=~\"$job\",instance=~\"$node\"}) by (actor_id, fragment_id, compute_node)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"group(actor_info{job=~\"$job\",instance=~\"$node\"}) by (actor_id, fragment_id, compute_node)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true},"indexByName":{"actor_id":0,"compute_node":2,"fragment_id":1}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Information about state tables. Column `materialized_view_id` is the id of the materialized view that this state table belongs to.","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":3,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true,"sortBy":[]},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name, table_type, materialized_view_id, fragment_id, compaction_group_id)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name, table_type, materialized_view_id, fragment_id, compaction_group_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"State Table Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true},"indexByName":{"compaction_group_id":5,"fragment_id":4,"materialized_view_id":3,"table_id":0,"table_name":1,"table_type":2}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Actor count per compute node","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":4,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true,"sortBy":[]},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"count(actor_info{job=~\"$job\",instance=~\"$node\"}) by (compute_node)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"count(actor_info{job=~\"$job\",instance=~\"$node\"}) by (compute_node)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Count (Group By Compute Node)","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true},"indexByName":{"compaction_group_id":5,"fragment_id":4,"materialized_view_id":3,"table_id":0,"table_name":1,"table_type":2}}}],"transparent":false,"type":"table"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Actor/Table Id Info","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":false,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":1},"height":null,"hideTimeOverride":false,"id":5,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Overview","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":2},"height":null,"hideTimeOverride":false,"id":6,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","query":"sum(rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of bytes read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":2},"height":null,"hideTimeOverride":false,"id":7,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum by (source_id, source_name, fragment_id)(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","query":"(sum by (source_id, source_name, fragment_id)(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of rows streamed into each sink per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":10},"height":null,"hideTimeOverride":false,"id":8,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id) * on(sink_id) group_left(sink_name) group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}}","metric":"","query":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id) * on(sink_id) group_left(sink_name) group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (sink_id, sink_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized view per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":10},"height":null,"hideTimeOverride":false,"id":9,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id) * on(table_id) group_left(table_name) group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mview {{table_id}} {{table_name}}","metric":"","query":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id) * on(table_id) group_left(table_name) group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The time that the data between two consecutive barriers gets fully processed, i.e. the computation results are made durable into materialized views or sink to external systems. This metric shows to users the freshness of materialized views.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":18},"height":null,"hideTimeOverride":false,"id":10,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_avg","metric":"","query":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Alerts in the system group by type:\n            - Too Many Barriers: there are too many uncommitted barriers generated. This means the streaming graph is stuck or under heavy load. Check 'Barrier Latency' panel.\n            - Recovery Triggered: cluster recovery is triggered. Check 'Errors by Type' / 'Node Count' panels.\n            - Lagging Version: the checkpointed or pinned version id is lagging behind the current version id. Check 'Hummock Manager' section in dev dashboard.\n            - Lagging Epoch: the pinned or safe epoch is lagging behind the current max committed epoch. Check 'Hummock Manager' section in dev dashboard.\n            - Lagging Compaction: there are too many files in L0. This can be caused by compactor failure or lag of compactor resource. Check 'Compaction' section in dev dashboard.\n            - Lagging Vacuum: there are too many stale files waiting to be cleaned. This can be caused by compactor failure or lag of compactor resource. Check 'Compaction' section in dev dashboard.\n            - Abnormal Meta Cache Memory: the meta cache memory usage is too large, exceeding the expected 10 percent.\n            - Abnormal Block Cache Memory: the block cache memory usage is too large, exceeding the expected 10 percent.\n            - Abnormal Uploading Memory Usage: uploading memory is more than 70 percent of the expected, and is about to spill.\n            - Write Stall: Compaction cannot keep up. Stall foreground write.\n            ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":18},"height":null,"hideTimeOverride":false,"id":11,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"} >= bool 200","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Too Many Barriers","metric":"","query":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"} >= bool 200","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > bool 0 + sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) > bool 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Recovery Triggered","metric":"","query":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > bool 0 + sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) > bool 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"((storage_current_version_id{job=~\"$job\",instance=~\"$node\"} - storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}) >= bool 100) + ((storage_current_version_id{job=~\"$job\",instance=~\"$node\"} - storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}) >= bool 100)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Version","metric":"","query":"((storage_current_version_id{job=~\"$job\",instance=~\"$node\"} - storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}) >= bool 100) + ((storage_current_version_id{job=~\"$job\",instance=~\"$node\"} - storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}) >= bool 100)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"((storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"} - storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}) >= bool 6553600000 unless + storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"} == 0)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Epoch","metric":"","query":"((storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"} - storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}) >= bool 6553600000 unless + storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"} == 0)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(label_replace(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}, 'L0', 'L0', 'level_index', '.*_L0') unless storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (L0) >= bool 200","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Compaction","metric":"","query":"sum(label_replace(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}, 'L0', 'L0', 'level_index', '.*_L0') unless storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (L0) >= bool 200","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"} >= bool 200","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Vacuum","metric":"","query":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"} >= bool 200","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_meta_cache_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 1.1","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Abnormal Meta Cache Memory","metric":"","query":"state_store_meta_cache_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 1.1","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_block_cache_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 1.1","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Abnormal Block Cache Memory","metric":"","query":"state_store_block_cache_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 1.1","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_uploading_memory_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 0.7","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Abnormal Uploading Memory Usage","metric":"","query":"state_store_uploading_memory_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 0.7","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_write_stop_compaction_groups{job=~\"$job\",instance=~\"$node\"} > bool 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Write Stall","metric":"","query":"storage_write_stop_compaction_groups{job=~\"$job\",instance=~\"$node\"} > bool 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Alerts","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Errors in the system group by type","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":26},"height":null,"hideTimeOverride":false,"id":12,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_compute_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, executor_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{executor_name}} (fragment_id={{fragment_id}})","metric":"","query":"sum(user_compute_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, executor_name, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_source_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{source_name}} (source_id={{source_id}} fragment_id={{fragment_id}})","metric":"","query":"sum(user_source_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, source_id, source_name, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_sink_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, sink_id, sink_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{sink_name}} (sink_id={{sink_id}} fragment_id={{fragment_id}})","metric":"","query":"sum(user_sink_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, sink_id, sink_name, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_status_is_up{job=~\"$job\",instance=~\"$node\"} == 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source error: source_id={{source_id}}, source_name={{source_name}} @ {{instance}}","metric":"","query":"source_status_is_up{job=~\"$job\",instance=~\"$node\"} == 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote storage error {{type}}: {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Errors","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":26},"height":null,"hideTimeOverride":false,"id":13,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Local mode","metric":"","query":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distributed mode","metric":"","query":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Query QPS","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of each type of RisingWave components alive.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":34},"height":null,"hideTimeOverride":false,"id":14,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_type}}","metric":"","query":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of active sessions in frontend nodes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":34},"height":null,"hideTimeOverride":false,"id":15,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Active Sessions","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":42},"height":null,"hideTimeOverride":false,"id":16,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The CPU usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":17,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of CPU cores per RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":18,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU Core Number","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"CPU","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":43},"height":null,"hideTimeOverride":false,"id":19,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The memory usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":20,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Memory","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":21,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (instance) + sum(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (instance) + sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage @ {{instance}}","metric":"","query":"sum(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (instance) + sum(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (instance) + sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Usage (Total)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":22,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(actor_memory_usage[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"streaming actor - {{actor_id}}","metric":"","query":"rate(actor_memory_usage[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage meta cache - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage block cache - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage write buffer - {{job}} @ {{instance}}","metric":"","query":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id) group_left(materialized_view_id) table_info) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized_view {{materialized_view_id}}","metric":"","query":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id) group_left(materialized_view_id) table_info) by (materialized_view_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Usage (Detailed)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Executor cache statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":23,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Join - cache miss - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Join - total lookups - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n appendonly - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n appendonly - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lookup executor - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lookup executor - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal join - cache miss - table_id {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal join - total lookups - table_id {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Materialize - cache hit count - table {{table_id}} - actor {{actor_id}}  {{instance}}","metric":"","query":"rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Materialize - total cache count - table {{table_id}} - actor {{actor_id}}  {{instance}}","metric":"","query":"rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":24,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id)) >=0 ","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"join executor cache miss ratio - - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","query":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id)) >=0 ","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0 ","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0 ","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n appendonly cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream lookup cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream temporal join cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialize executor cache miss ratio - table {{table_id}} - actor {{actor_id}}  {{instance}}","metric":"","query":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Miss Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Storage cache statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":25,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"memory cache - {{table_id}} @ {{type}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_sst_store_block_request_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total_meta_miss_count - {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Storage bloom filter statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":26,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_read_req_check_bloom_filter_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter total - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_read_req_check_bloom_filter_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_read_req_positive_but_non_exist_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter false positive  - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_read_req_positive_but_non_exist_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Bloom Filer","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Storage file cache statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":27,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(file_cache_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"file cache {{op}} @ {{instance}}","metric":"","query":"sum(rate(file_cache_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(file_cache_miss{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"file cache miss @ {{instance}}","metric":"","query":"sum(rate(file_cache_miss{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage File Cache","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Memory","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":44},"height":null,"hideTimeOverride":false,"id":28,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Send/Recv throughput per node for streaming exchange","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":29,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Send @ {{instance}}","metric":"","query":"sum(rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Recv @ {{instance}}","metric":"","query":"sum(rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Streming Remote Exchange (Bytes/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The remote storage read/write throughput per node","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":30,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{instance}}","metric":"","query":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{instance}}","metric":"","query":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Remote I/O (Bytes/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"row"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":31,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{query_id}} : {{source_stage_id}}.{{source_task_id}} -> {{target_stage_id}}.{{target_task_id}}","metric":"","query":"batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Exchange Recv (Rows/s)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Network","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":45},"height":null,"hideTimeOverride":false,"id":32,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"\n                    Objects are classified into 3 groups:\n                    - not referenced by versions: these object are being deleted from object store.\n                    - referenced by non-current versions: these objects are stale (not in the latest version), but those old versions may still be in use (e.g. long-running pinning). Thus those objects cannot be deleted at the moment.\n                    - referenced by current version: these objects are in the latest version.\n                    ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":33,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","query":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","query":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","query":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The storage size of each materialized view","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":34,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_materialized_view_stats{metric='materialized_view_total_size',job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{metric}}, mv id - {{table_id}} ","metric":"","query":"storage_materialized_view_stats{metric='materialized_view_total_size',job=~\"$job\",instance=~\"$node\"}/1024","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"\n                    Objects are classified into 3 groups:\n                    - not referenced by versions: these object are being deleted from object store.\n                    - referenced by non-current versions: these objects are stale (not in the latest version), but those old versions may still be in use (e.g. long-running pinning). Thus those objects cannot be deleted at the moment.\n                    - referenced by current version: these objects are in the latest version.\n                    ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":35,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","query":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","query":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","query":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of bytes that have been written by compaction.Flush refers to the process of compacting Memtables to SSTables at Level 0.Compaction refers to the process of compacting SSTables at one level to another level.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":36,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Compaction - {{job}}","metric":"","query":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush - {{job}}","metric":"","query":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Bytes","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The remote storage read/write throughput","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":37,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}}","metric":"","query":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}}","metric":"","query":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Remote I/O (Bytes/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Size statistics for checkpoint","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":38,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{job}}","metric":"","query":"sum by(le, job) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Checkpoint Size","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Storage","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":46},"height":null,"hideTimeOverride":false,"id":39,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":40,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_name}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of bytes read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":41,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum by (source_id)(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_id}}","metric":"","query":"(sum by (source_id)(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":42,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_source_backfill_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","query":"sum(rate(stream_source_backfill_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Backfill Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized executor actor per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":43,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_executor_row_count{executor_identity=~\".*MaterializeExecutor.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) * on(actor_id) group_left(materialized_view_id, table_name) (group(table_info{table_type=~\"MATERIALIZED_VIEW\",job=~\"$job\",instance=~\"$node\"}) by (actor_id, materialized_view_id, table_name))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized view {{table_name}} table_id {{materialized_view_id}}","metric":"","query":"sum(rate(stream_executor_row_count{executor_identity=~\".*MaterializeExecutor.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) * on(actor_id) group_left(materialized_view_id, table_name) (group(table_info{table_type=~\"MATERIALIZED_VIEW\",job=~\"$job\",instance=~\"$node\"}) by (actor_id, materialized_view_id, table_name))) by (materialized_view_id, table_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been read from the backfill operator used by MV on MV","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":44,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_snapshot_read_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Read Snapshot - table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_backfill_snapshot_read_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_upstream_output_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Upstream - table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_backfill_upstream_output_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"We first record the total blocking duration(ns) of output buffer of each actor. It shows how much time it takes an actor to process a message, i.e. a barrier, a watermark or rows of data, on average. Then we divide this duration by 1 second and show it as a percentage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":45,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, downstream_fragment_id) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}->{{downstream_fragment_id}}","metric":"","query":"avg(rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, downstream_fragment_id) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Output Blocking Time Ratio (Backpressure)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":47},"height":null,"hideTimeOverride":false,"id":46,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":47,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of running query in distributed execution mode","metric":"","query":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Running query in distributed execution mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":48,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of rejected query in distributed execution mode","metric":"","query":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Rejected query in distributed execution mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":49,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of completed query in distributed execution mode","metric":"","query":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Completed query in distributed execution mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":50,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency in Distributed Execution Mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":51,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency in Local Execution Mode","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Batch","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":52,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":53,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(connector_source_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_type}} @ {{source_id}}","metric":"","query":"rate(connector_source_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Source Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":54,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(connector_sink_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector_type}} @ {{sink_id}}","metric":"","query":"rate(connector_sink_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Sink Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Connector Node","transformations":[],"transparent":false,"type":"row"}],"refresh":"","rows":[],"schemaVersion":12,"sharedCrosshair":true,"style":"dark","tags":["risingwave"],"templating":{"list":[{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, instance)","description":"Reporting instance of the metric","hide":0,"includeAll":true,"label":"Node","multi":true,"name":"node","options":[],"query":{"query":"label_values(process_cpu_seconds_total, instance)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, job)","description":"Reporting job of the metric","hide":0,"includeAll":true,"label":"Job","multi":true,"name":"job","options":[],"query":{"query":"label_values(process_cpu_seconds_total, job)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"}]},"time":{"from":"now-30m","to":"now"},"timepicker":{"hidden":false,"nowDelay":null,"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone":"browser","title":"risingwave_dashboard","uid":"Fcy3uV1nz","version":0}
+{"__inputs":[],"annotations":{"list":[]},"description":"RisingWave Dashboard","editable":true,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"panels":[{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":1,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Information about actors","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]}},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":2,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true},"repeat":null,"repeatDirection":null,"span":6,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"group(actor_info{job=~\"$job\",instance=~\"$node\"}) by (actor_id, fragment_id, compute_node)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"group(actor_info{job=~\"$job\",instance=~\"$node\"}) by (actor_id, fragment_id, compute_node)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true},"indexByName":{"actor_id":0,"compute_node":2,"fragment_id":1}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Information about state tables. Column `materialized_view_id` is the id of the materialized view that this state table belongs to.","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]}},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":3,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true},"repeat":null,"repeatDirection":null,"span":6,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name, table_type, materialized_view_id, fragment_id, compaction_group_id)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name, table_type, materialized_view_id, fragment_id, compaction_group_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"State Table Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true},"indexByName":{"compaction_group_id":5,"fragment_id":4,"materialized_view_id":3,"table_id":0,"table_name":1,"table_type":2}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Actor count per compute node","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]}},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":4,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true},"repeat":null,"repeatDirection":null,"span":6,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"count(actor_info{job=~\"$job\",instance=~\"$node\"}) by (compute_node)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"count(actor_info{job=~\"$job\",instance=~\"$node\"}) by (compute_node)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Count (Group By Compute Node)","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true},"indexByName":{"compaction_group_id":5,"fragment_id":4,"materialized_view_id":3,"table_id":0,"table_name":1,"table_type":2}}}],"transparent":false,"type":"table"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Actor/Table Id Info","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":false,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":1},"height":null,"hideTimeOverride":false,"id":5,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Overview","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":2},"height":null,"hideTimeOverride":false,"id":6,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","query":"sum(rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of bytes read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":2},"height":null,"hideTimeOverride":false,"id":7,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum by (source_id, source_name, fragment_id)(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","query":"(sum by (source_id, source_name, fragment_id)(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of rows streamed into each sink per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":10},"height":null,"hideTimeOverride":false,"id":8,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id) * on(sink_id) group_left(sink_name) group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}}","metric":"","query":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id) * on(sink_id) group_left(sink_name) group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (sink_id, sink_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized view per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":10},"height":null,"hideTimeOverride":false,"id":9,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id) * on(table_id) group_left(table_name) group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mview {{table_id}} {{table_name}}","metric":"","query":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id) * on(table_id) group_left(table_name) group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The time that the data between two consecutive barriers gets fully processed, i.e. the computation results are made durable into materialized views or sink to external systems. This metric shows to users the freshness of materialized views.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":18},"height":null,"hideTimeOverride":false,"id":10,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_avg","metric":"","query":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Alerts in the system group by type:\n            - Too Many Barriers: there are too many uncommitted barriers generated. This means the streaming graph is stuck or under heavy load. Check 'Barrier Latency' panel.\n            - Recovery Triggered: cluster recovery is triggered. Check 'Errors by Type' / 'Node Count' panels.\n            - Lagging Version: the checkpointed or pinned version id is lagging behind the current version id. Check 'Hummock Manager' section in dev dashboard.\n            - Lagging Epoch: the pinned or safe epoch is lagging behind the current max committed epoch. Check 'Hummock Manager' section in dev dashboard.\n            - Lagging Compaction: there are too many files in L0. This can be caused by compactor failure or lag of compactor resource. Check 'Compaction' section in dev dashboard.\n            - Lagging Vacuum: there are too many stale files waiting to be cleaned. This can be caused by compactor failure or lag of compactor resource. Check 'Compaction' section in dev dashboard.\n            - Abnormal Meta Cache Memory: the meta cache memory usage is too large, exceeding the expected 10 percent.\n            - Abnormal Block Cache Memory: the block cache memory usage is too large, exceeding the expected 10 percent.\n            - Abnormal Uploading Memory Usage: uploading memory is more than 70 percent of the expected, and is about to spill.\n            - Write Stall: Compaction cannot keep up. Stall foreground write.\n            ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":18},"height":null,"hideTimeOverride":false,"id":11,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"} >= bool 200","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Too Many Barriers","metric":"","query":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"} >= bool 200","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > bool 0 + sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) > bool 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Recovery Triggered","metric":"","query":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > bool 0 + sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) > bool 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"((storage_current_version_id{job=~\"$job\",instance=~\"$node\"} - storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}) >= bool 100) + ((storage_current_version_id{job=~\"$job\",instance=~\"$node\"} - storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}) >= bool 100)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Version","metric":"","query":"((storage_current_version_id{job=~\"$job\",instance=~\"$node\"} - storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}) >= bool 100) + ((storage_current_version_id{job=~\"$job\",instance=~\"$node\"} - storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}) >= bool 100)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"((storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"} - storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}) >= bool 6553600000 unless + storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"} == 0)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Epoch","metric":"","query":"((storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"} - storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}) >= bool 6553600000 unless + storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"} == 0)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(label_replace(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}, 'L0', 'L0', 'level_index', '.*_L0') unless storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (L0) >= bool 200","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Compaction","metric":"","query":"sum(label_replace(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}, 'L0', 'L0', 'level_index', '.*_L0') unless storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (L0) >= bool 200","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"} >= bool 200","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Vacuum","metric":"","query":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"} >= bool 200","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_meta_cache_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 1.1","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Abnormal Meta Cache Memory","metric":"","query":"state_store_meta_cache_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 1.1","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_block_cache_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 1.1","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Abnormal Block Cache Memory","metric":"","query":"state_store_block_cache_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 1.1","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_uploading_memory_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 0.7","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Abnormal Uploading Memory Usage","metric":"","query":"state_store_uploading_memory_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 0.7","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_write_stop_compaction_groups{job=~\"$job\",instance=~\"$node\"} > bool 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Write Stall","metric":"","query":"storage_write_stop_compaction_groups{job=~\"$job\",instance=~\"$node\"} > bool 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Alerts","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Errors in the system group by type","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":26},"height":null,"hideTimeOverride":false,"id":12,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_compute_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, executor_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{executor_name}} (fragment_id={{fragment_id}})","metric":"","query":"sum(user_compute_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, executor_name, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_source_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{source_name}} (source_id={{source_id}} fragment_id={{fragment_id}})","metric":"","query":"sum(user_source_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, source_id, source_name, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_sink_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, sink_id, sink_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{sink_name}} (sink_id={{sink_id}} fragment_id={{fragment_id}})","metric":"","query":"sum(user_sink_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, sink_id, sink_name, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_status_is_up{job=~\"$job\",instance=~\"$node\"} == 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source error: source_id={{source_id}}, source_name={{source_name}} @ {{instance}}","metric":"","query":"source_status_is_up{job=~\"$job\",instance=~\"$node\"} == 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote storage error {{type}}: {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Errors","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":26},"height":null,"hideTimeOverride":false,"id":13,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Local mode","metric":"","query":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distributed mode","metric":"","query":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Query QPS","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of each type of RisingWave components alive.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":34},"height":null,"hideTimeOverride":false,"id":14,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_type}}","metric":"","query":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of active sessions in frontend nodes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":34},"height":null,"hideTimeOverride":false,"id":15,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Active Sessions","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":42},"height":null,"hideTimeOverride":false,"id":16,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The CPU usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":17,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of CPU cores per RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":18,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU Core Number","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"CPU","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":43},"height":null,"hideTimeOverride":false,"id":19,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The memory usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":20,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Memory","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":21,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (instance) + sum(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (instance) + sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage @ {{instance}}","metric":"","query":"sum(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (instance) + sum(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (instance) + sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Usage (Total)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":22,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(actor_memory_usage[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"streaming actor - {{actor_id}}","metric":"","query":"rate(actor_memory_usage[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage meta cache - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage block cache - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage write buffer - {{job}} @ {{instance}}","metric":"","query":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id) group_left(materialized_view_id) table_info) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized_view {{materialized_view_id}}","metric":"","query":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id) group_left(materialized_view_id) table_info) by (materialized_view_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Usage (Detailed)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Executor cache statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":23,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Join - cache miss - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Join - total lookups - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n appendonly - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n appendonly - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lookup executor - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lookup executor - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal join - cache miss - table_id {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal join - total lookups - table_id {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Materialize - cache hit count - table {{table_id}} - actor {{actor_id}}  {{instance}}","metric":"","query":"rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Materialize - total cache count - table {{table_id}} - actor {{actor_id}}  {{instance}}","metric":"","query":"rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":24,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id)) >=0 ","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"join executor cache miss ratio - - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","query":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id)) >=0 ","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0 ","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0 ","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n appendonly cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream lookup cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream temporal join cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialize executor cache miss ratio - table {{table_id}} - actor {{actor_id}}  {{instance}}","metric":"","query":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Miss Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Storage cache statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":25,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"memory cache - {{table_id}} @ {{type}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_sst_store_block_request_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total_meta_miss_count - {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Storage bloom filter statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":26,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_read_req_check_bloom_filter_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter total - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_read_req_check_bloom_filter_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_read_req_positive_but_non_exist_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter false positive  - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_read_req_positive_but_non_exist_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Bloom Filer","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Storage file cache statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":27,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(file_cache_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"file cache {{op}} @ {{instance}}","metric":"","query":"sum(rate(file_cache_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(file_cache_miss{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"file cache miss @ {{instance}}","metric":"","query":"sum(rate(file_cache_miss{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage File Cache","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Memory","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":44},"height":null,"hideTimeOverride":false,"id":28,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Send/Recv throughput per node for streaming exchange","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":29,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Send @ {{instance}}","metric":"","query":"sum(rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Recv @ {{instance}}","metric":"","query":"sum(rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Streming Remote Exchange (Bytes/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The remote storage read/write throughput per node","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":30,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{instance}}","metric":"","query":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{instance}}","metric":"","query":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Remote I/O (Bytes/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"row"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":31,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{query_id}} : {{source_stage_id}}.{{source_task_id}} -> {{target_stage_id}}.{{target_task_id}}","metric":"","query":"batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Exchange Recv (Rows/s)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Network","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":45},"height":null,"hideTimeOverride":false,"id":32,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"\n                    Objects are classified into 3 groups:\n                    - not referenced by versions: these object are being deleted from object store.\n                    - referenced by non-current versions: these objects are stale (not in the latest version), but those old versions may still be in use (e.g. long-running pinning). Thus those objects cannot be deleted at the moment.\n                    - referenced by current version: these objects are in the latest version.\n                    ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":33,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","query":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","query":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","query":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The storage size of each materialized view","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":34,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_materialized_view_stats{metric='materialized_view_total_size',job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{metric}}, mv id - {{table_id}} ","metric":"","query":"storage_materialized_view_stats{metric='materialized_view_total_size',job=~\"$job\",instance=~\"$node\"}/1024","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"\n                    Objects are classified into 3 groups:\n                    - not referenced by versions: these object are being deleted from object store.\n                    - referenced by non-current versions: these objects are stale (not in the latest version), but those old versions may still be in use (e.g. long-running pinning). Thus those objects cannot be deleted at the moment.\n                    - referenced by current version: these objects are in the latest version.\n                    ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":35,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","query":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","query":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","query":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of bytes that have been written by compaction.Flush refers to the process of compacting Memtables to SSTables at Level 0.Compaction refers to the process of compacting SSTables at one level to another level.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":36,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Compaction - {{job}}","metric":"","query":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush - {{job}}","metric":"","query":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Bytes","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The remote storage read/write throughput","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":37,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}}","metric":"","query":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}}","metric":"","query":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Remote I/O (Bytes/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Size statistics for checkpoint","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":38,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{job}}","metric":"","query":"sum by(le, job) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Checkpoint Size","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Storage","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":46},"height":null,"hideTimeOverride":false,"id":39,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":40,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_name}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of bytes read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":41,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum by (source_id)(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_id}}","metric":"","query":"(sum by (source_id)(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":42,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_source_backfill_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","query":"sum(rate(stream_source_backfill_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Backfill Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized executor actor per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":43,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_executor_row_count{executor_identity=~\".*MaterializeExecutor.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) * on(actor_id) group_left(materialized_view_id, table_name) (group(table_info{table_type=~\"MATERIALIZED_VIEW\",job=~\"$job\",instance=~\"$node\"}) by (actor_id, materialized_view_id, table_name))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized view {{table_name}} table_id {{materialized_view_id}}","metric":"","query":"sum(rate(stream_executor_row_count{executor_identity=~\".*MaterializeExecutor.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) * on(actor_id) group_left(materialized_view_id, table_name) (group(table_info{table_type=~\"MATERIALIZED_VIEW\",job=~\"$job\",instance=~\"$node\"}) by (actor_id, materialized_view_id, table_name))) by (materialized_view_id, table_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been read from the backfill operator used by MV on MV","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":44,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_snapshot_read_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Read Snapshot - table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_backfill_snapshot_read_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_upstream_output_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Upstream - table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_backfill_upstream_output_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"We first record the total blocking duration(ns) of output buffer of each actor. It shows how much time it takes an actor to process a message, i.e. a barrier, a watermark or rows of data, on average. Then we divide this duration by 1 second and show it as a percentage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":45,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, downstream_fragment_id) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}->{{downstream_fragment_id}}","metric":"","query":"avg(rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, downstream_fragment_id) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Output Blocking Time Ratio (Backpressure)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":47},"height":null,"hideTimeOverride":false,"id":46,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":47,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of running query in distributed execution mode","metric":"","query":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Running query in distributed execution mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":48,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of rejected query in distributed execution mode","metric":"","query":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Rejected query in distributed execution mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":49,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of completed query in distributed execution mode","metric":"","query":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Completed query in distributed execution mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":50,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency in Distributed Execution Mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":51,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency in Local Execution Mode","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Batch","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":52,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":53,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(connector_source_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_type}} @ {{source_id}}","metric":"","query":"rate(connector_source_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Source Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":54,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(connector_sink_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector_type}} @ {{sink_id}}","metric":"","query":"rate(connector_sink_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Sink Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Connector Node","transformations":[],"transparent":false,"type":"row"}],"refresh":"","rows":[],"schemaVersion":12,"sharedCrosshair":true,"style":"dark","tags":["risingwave"],"templating":{"list":[{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, instance)","description":"Reporting instance of the metric","hide":0,"includeAll":true,"label":"Node","multi":true,"name":"node","options":[],"query":{"query":"label_values(process_cpu_seconds_total, instance)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, job)","description":"Reporting job of the metric","hide":0,"includeAll":true,"label":"Job","multi":true,"name":"job","options":[],"query":{"query":"label_values(process_cpu_seconds_total, job)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"}]},"time":{"from":"now-30m","to":"now"},"timepicker":{"hidden":false,"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone":"browser","title":"risingwave_dashboard","uid":"Fcy3uV1nz","version":0}
diff --git a/docker/docker-compose-distributed-etcd.yml b/docker/docker-compose-distributed-etcd.yml
index 16382a54a2b73..1e23484c22f72 100644
--- a/docker/docker-compose-distributed-etcd.yml
+++ b/docker/docker-compose-distributed-etcd.yml
@@ -1,7 +1,7 @@
 ---
 version: "3"
 x-image: &image
-  image: ${RW_IMAGE:-risingwavelabs/risingwave:v1.10.0-rc.1}
+  image: ${RW_IMAGE:-risingwavelabs/risingwave:v1.10.0}
 services:
   compactor-0:
     <<: *image
diff --git a/docker/docker-compose-distributed.yml b/docker/docker-compose-distributed.yml
index 843d15c39cfde..8de40728fd963 100644
--- a/docker/docker-compose-distributed.yml
+++ b/docker/docker-compose-distributed.yml
@@ -1,6 +1,6 @@
 ---
 x-image: &image
-  image: ${RW_IMAGE:-risingwavelabs/risingwave:v1.10.0-rc.1}
+  image: ${RW_IMAGE:-risingwavelabs/risingwave:v1.10.0}
 services:
   compactor-0:
     <<: *image
diff --git a/docker/docker-compose-etcd.yml b/docker/docker-compose-etcd.yml
index 5cca2a704d9b7..ef444fa2f0d82 100644
--- a/docker/docker-compose-etcd.yml
+++ b/docker/docker-compose-etcd.yml
@@ -1,7 +1,7 @@
 ---
 version: "3"
 x-image: &image
-  image: ${RW_IMAGE:-risingwavelabs/risingwave:v1.10.0-rc.1}
+  image: ${RW_IMAGE:-risingwavelabs/risingwave:v1.10.0}
 services:
   risingwave-standalone:
     <<: *image
diff --git a/docker/docker-compose-with-azblob.yml b/docker/docker-compose-with-azblob.yml
index ed056d9ad1c53..7c6a30e1f336c 100644
--- a/docker/docker-compose-with-azblob.yml
+++ b/docker/docker-compose-with-azblob.yml
@@ -1,6 +1,6 @@
 ---
 x-image: &image
-  image: ${RW_IMAGE:-risingwavelabs/risingwave:v1.10.0-rc.1}
+  image: ${RW_IMAGE:-risingwavelabs/risingwave:v1.10.0}
 services:
   risingwave-standalone:
     <<: *image
diff --git a/docker/docker-compose-with-gcs.yml b/docker/docker-compose-with-gcs.yml
index 28e52286df395..9327e6b4ee8cb 100644
--- a/docker/docker-compose-with-gcs.yml
+++ b/docker/docker-compose-with-gcs.yml
@@ -1,6 +1,6 @@
 ---
 x-image: &image
-  image: ${RW_IMAGE:-risingwavelabs/risingwave:v1.10.0-rc.1}
+  image: ${RW_IMAGE:-risingwavelabs/risingwave:v1.10.0}
 services:
   risingwave-standalone:
     <<: *image
diff --git a/docker/docker-compose-with-local-fs.yml b/docker/docker-compose-with-local-fs.yml
index d6f0699c22b28..d52a2adc911fd 100644
--- a/docker/docker-compose-with-local-fs.yml
+++ b/docker/docker-compose-with-local-fs.yml
@@ -1,6 +1,6 @@
 ---
 x-image: &image
-  image: ${RW_IMAGE:-risingwavelabs/risingwave:v1.10.0-rc.1}
+  image: ${RW_IMAGE:-risingwavelabs/risingwave:v1.10.0}
 services:
   risingwave-standalone:
     <<: *image
diff --git a/docker/docker-compose-with-obs.yml b/docker/docker-compose-with-obs.yml
index 72201df7e7dd6..d6beb4f86e89e 100644
--- a/docker/docker-compose-with-obs.yml
+++ b/docker/docker-compose-with-obs.yml
@@ -1,6 +1,6 @@
 ---
 x-image: &image
-  image: ${RW_IMAGE:-risingwavelabs/risingwave:v1.10.0-rc.1}
+  image: ${RW_IMAGE:-risingwavelabs/risingwave:v1.10.0}
 services:
   risingwave-standalone:
     <<: *image
diff --git a/docker/docker-compose-with-oss.yml b/docker/docker-compose-with-oss.yml
index 1fba5ea52f348..74e4ec15d8f3e 100644
--- a/docker/docker-compose-with-oss.yml
+++ b/docker/docker-compose-with-oss.yml
@@ -1,6 +1,6 @@
 ---
 x-image: &image
-  image: ${RW_IMAGE:-risingwavelabs/risingwave:v1.10.0-rc.1}
+  image: ${RW_IMAGE:-risingwavelabs/risingwave:v1.10.0}
 services:
   risingwave-standalone:
     <<: *image
diff --git a/docker/docker-compose-with-s3.yml b/docker/docker-compose-with-s3.yml
index a0cdcf5ef73de..c6ca1a885b448 100644
--- a/docker/docker-compose-with-s3.yml
+++ b/docker/docker-compose-with-s3.yml
@@ -1,6 +1,6 @@
 ---
 x-image: &image
-  image: ${RW_IMAGE:-risingwavelabs/risingwave:v1.10.0-rc.1}
+  image: ${RW_IMAGE:-risingwavelabs/risingwave:v1.10.0}
 services:
   risingwave-standalone:
     <<: *image
diff --git a/docker/docker-compose-with-sqlite.yml b/docker/docker-compose-with-sqlite.yml
index 0c3a04a661f33..a4b008c1374cd 100644
--- a/docker/docker-compose-with-sqlite.yml
+++ b/docker/docker-compose-with-sqlite.yml
@@ -1,6 +1,6 @@
 ---
 x-image: &image
-  image: ${RW_IMAGE:-risingwavelabs/risingwave:v1.10.0-rc.1}
+  image: ${RW_IMAGE:-risingwavelabs/risingwave:v1.10.0}
 services:
   risingwave-standalone:
     <<: *image
diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
index dc9ad11f0eebc..c1681ff658765 100644
--- a/docker/docker-compose.yml
+++ b/docker/docker-compose.yml
@@ -1,6 +1,6 @@
 ---
 x-image: &image
-  image: ${RW_IMAGE:-risingwavelabs/risingwave:v1.10.0-rc.1}
+  image: ${RW_IMAGE:-risingwavelabs/risingwave:v1.10.0}
 services:
   risingwave-standalone:
     <<: *image
@@ -59,6 +59,7 @@ services:
       # If ENABLE_TELEMETRY is not set, telemetry will start by default
       ENABLE_TELEMETRY: ${ENABLE_TELEMETRY:-true}
       RW_TELEMETRY_TYPE: ${RW_TELEMETRY_TYPE:-"docker-compose"}
+      RW_SECRET_STORE_PRIVATE_KEY_HEX: ${RW_SECRET_STORE_PRIVATE_KEY_HEX:-0123456789abcdef}
     container_name: risingwave-standalone
     healthcheck:
       test:
diff --git a/docs/README.md b/docs/README.md
index e905cea7849ea..f371d9bda8f47 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -2,25 +2,5 @@
 
 This directory contains RisingWave design documents that are intended to be used by contributors to understand our development process, and how we design and implement RisingWave. To learn about how to use RisingWave, check out the [RisingWave user documentation](https://www.risingwave.dev).
 
-## Developer guide
-
-After you learn about the basics of RisingWave, take a look at our [developer guide](https://risingwavelabs.github.io/risingwave/) to get up to speed with the development process.
-
-## Table of Contents
-
-* [Architecture Design](./architecture-design.md)
-* [An Overview of RisingWave Streaming Engine](./streaming-overview.md)
-* [An Overview of RisingWave State Store](./state-store-overview.md)
-* [Meta Service](./meta-service.md)
-* [Create MView on Top of MView](./mv-on-mv.md)
-* [Checkpoint](./checkpoint.md)
-* [Design of Data Source](./data-source.md)
-* [Data Model and Encoding](./data-model-and-encoding.md)
-* [Design of Batch Local Execution Mode](./batch-local-execution-mode.md)
-* [Consistent Hash](./consistent-hash.md)
-* [Build RisingWave with Multiple Object Storage Backends](./multi-object-store.md)
-* [Backfill](./backfill.md)
-
-## Images
-
-We recommend that you use [draw.io](https://app.diagrams.net/) to draw illustrations and export as SVG images, with "include a copy of my diagram" selected for further editing.
+- `/dev` contains the source code for the [RisingWave Developer Guide](https://risingwavelabs.github.io/risingwave/)
+- `/rustdoc` contains source code for the [crate level documentation](https://risingwavelabs.github.io/risingwave/rustdoc)
diff --git a/docs/dev/README.md b/docs/dev/README.md
index e19f10c08e3c3..7e47920d49400 100644
--- a/docs/dev/README.md
+++ b/docs/dev/README.md
@@ -28,3 +28,7 @@ including the `<!-- toc -->` marker at the place where you want the TOC.
 
 We use `mdbook-linkcheck` to validate URLs included in our documentation.
 `linkcheck` will be run automatically when you build with the instructions in the section above.
+
+## Images
+
+We recommend that you use [draw.io](https://app.diagrams.net/) to draw illustrations and export as SVG images, with "include a copy of my diagram" selected for further editing.
diff --git a/docs/dev/src/SUMMARY.md b/docs/dev/src/SUMMARY.md
index 76cf57c007c23..fcc82a615c410 100644
--- a/docs/dev/src/SUMMARY.md
+++ b/docs/dev/src/SUMMARY.md
@@ -8,13 +8,15 @@
 # Building and debugging RisingWave
 
 - [Building and Running](./build-and-run/intro.md)
+    - [Profiles](./build-and-run/profiles.md)
 - [Testing](./tests/intro.md)
 - [Debugging](./debugging.md)
 - [Observability](./observability.md)
+    - [Metrics](./metrics.md)
 
 ---
 
-# Benchmarking and Profiling
+# Benchmarking and profiling
 
 - [CPU Profiling](./benchmark-and-profile/cpu-profiling.md)
 - [Memory (Heap) Profiling](./benchmark-and-profile/memory-profiling.md)
@@ -25,8 +27,28 @@
 # Specialized topics
 
 - [Develop Connectors](./connector/intro.md)
+    - [Source](./connector/source.md)
 - [Continuous Integration](./ci.md)
 
+---
+
+# Design docs
+
+- [Architecture Design](./design/architecture-design.md)
+- [Streaming Engine](./design/streaming-overview.md)
+    - [Checkpoint](./design/checkpoint.md)
+    - [Aggregation](./design/aggregation.md)
+    - [MView on Top of MView](./design/mv-on-mv.md)
+    - [Backfill](./design/backfill.md)
+- [State Store](./design/state-store-overview.md)
+    - [Shared Buffer](./design/shared-buffer.md)
+    - [Relational Table](./design/relational-table.md)
+    - [Multiple Object Storage Backends](./design/multi-object-store.md)
+- [Meta Service](./design/meta-service.md)
+- [Data Model and Encoding](./design/data-model-and-encoding.md)
+- [Batch Local Execution Mode](./design/batch-local-execution-mode.md)
+- [Consistent Hash](./design/consistent-hash.md)
+- [Keys](./design/keys.md)
 <!--
 
 TODO:
diff --git a/docs/dev/src/build-and-run/profiles.md b/docs/dev/src/build-and-run/profiles.md
new file mode 100644
index 0000000000000..99388d93b78b0
--- /dev/null
+++ b/docs/dev/src/build-and-run/profiles.md
@@ -0,0 +1,70 @@
+# Build Profiles
+
+RisingWave uses Cargo profiles to manage build settings. To briefly introduce Cargo profiles, here is a snippet from the [Cargo References](https://doc.rust-lang.org/cargo/reference/profiles.html):
+
+> Profiles provide a way to alter the compiler settings, influencing things like optimizations and debugging symbols.
+>
+> Cargo has 4 built-in profiles: `dev`, `release`, `test`, and `bench`. The profile is automatically chosen based on which command is being run if a profile is not specified on the command-line. In addition to the built-in profiles, custom user-defined profiles can also be specified.
+
+All profiles talked in this document are defined in the `Cargo.toml` file of the root directory of the project. Please always refer to it for the most up-to-date information.
+
+## Built-in Profiles
+
+RisingWave tweaks some settings of the built-in profiles to better fit its needs, in the sections of `[profile.<built-in-profile>]` in `Cargo.toml`. For example,
+
+- `dev`: for local development and testing
+
+  - completely disables [LTO](https://doc.rust-lang.org/cargo/reference/profiles.html#lto) to speed up the build time
+
+- `release`: for local testing with near-production performance
+
+  - completely disables LTO to speed up the build time
+  - embeds full debug information to help with debugging in production
+
+## Custom Profiles
+
+RisingWave also defines some custom profiles that inherit from the built-in profiles, in the sections of `[profile.<custom-profile>]` in `Cargo.toml`. For example,
+
+- `production`: for distribution and production deployment
+
+  - inherits from `release`
+  - enables aggressive code optimizations (like LTO) for maximum performance, at the cost of significantly increased build time
+
+- `ci-dev`: for `pull-request` pipelines in CI
+
+  - inherits from `dev`
+  - tweaks some settings to reduce the build time and binary size
+  - enables code optimizations for 3rd-party dependencies to improve CI performance
+
+- `ci-release`: for `main` and `main-cron` pipelines in CI
+
+  - inherits from `release`
+  - tweaks some settings to reduce the build time and binary size
+  - enables more runtime checks (like debug assertions and overflow checks) to catch potential bugs
+
+- `ci-sim`: for `madsim` simulation tests in CI
+  - similar to `ci-dev`
+  - enables slight code optimizations for all crates to improve CI performance under single-threaded madsim execution
+
+## Comparisons
+
+To give a better idea of the differences between the profiles, here is a matrix comparing the profiles:
+
+| Profile      | Debug Info     | `cfg(debug_assertions)` | Performance | Build Time |
+| ------------ | -------------- | ----------------------- | ----------- | ---------- |
+| `dev`        | Full           | `true`                  | Bad         | Fastest    |
+| `release`    | Full           | `false`                 | Good        | Slow       |
+| `production` | Full           | `false`                 | Best        | Slowest    |
+| `ci-dev`     | Backtrace only | `true`                  | Medium      | Fast       |
+| `ci-release` | Backtrace only | `true`                  | Good        | Slow       |
+| `ci-sim`     | Backtrace only | `true`                  | Medium      | Medium     |
+
+Some miscellaneous notes:
+
+- Compared to "Backtrace only", "Full" debug information additionally includes the capability to attach a debugger at runtime or on core dumps, to inspect variables and stack traces.
+- There are also other subtle differences like incremental compilation settings, overflow checks, and more. They are not listed here for brevity.
+- `cfg(debug_assertions)` can be roughly used to determine whether it's a production environment or not. Note that even though `ci-release` contains `release` in its name, the `debug_assertions` are still enabled.
+
+## Choose a Profile
+
+By default, RisingWave (and RiseDev) uses the `dev` profile for local development and testing. To use `release` profile instead, you can set the corresponding configuration entry by running `risedev configure`. Other profiles are for their specific use cases and are not meant to be used directly by developers, thus not available with RiseDev.
diff --git a/docs/dev/src/ci.md b/docs/dev/src/ci.md
index 8401737660557..1a34374cc3241 100644
--- a/docs/dev/src/ci.md
+++ b/docs/dev/src/ci.md
@@ -19,3 +19,23 @@ To run `e2e-test` and `e2e-source-test` for `main-cron` in your pull request:
 1. Add `ci/run-e2e-test`.
 2. Add `ci/run-e2e-source-tests`.
 3. Add `ci/main-cron/run-selected` to skip all other steps which were not selected with `ci/run-xxx`.
+
+## Main Cron Bisect Guide
+
+1. Create a new build via [buildkite](https://buildkite.com/risingwavelabs/main-cron-bisect/builds/#new)
+2. Add the following environment variables:
+   - `GOOD_COMMIT`: The good commit hash.
+   - `BAD_COMMIT`: The bad commit hash.
+   - `BISECT_BRANCH`: The branch name where the bisect will be performed.
+   - `CI_STEPS`: The `CI_STEPS` to run during the bisect. Separate multiple steps with a comma.
+     - You can check the labels for this in [main-cron.yml](https://github.com/risingwavelabs/risingwave/blob/main/ci/workflows/main-cron.yml),
+       under the conditions for each step.
+
+Example you can try on [buildkite](https://buildkite.com/risingwavelabs/main-cron-bisect/builds/#new):
+- Environment variables:
+  ```
+  GOOD_COMMIT=29791ddf16fdf2c2e83ad3a58215f434e610f89a
+  BAD_COMMIT=7f36bf17c1d19a1e6b2cdb90491d3c08ae8b0004
+  BISECT_BRANCH=kwannoel/test-bisect
+  CI_STEPS="test-bisect,disable-build"
+  ```
\ No newline at end of file
diff --git a/docs/dev/src/connector/intro.md b/docs/dev/src/connector/intro.md
index 88d487fe96c47..e9903175c5f61 100644
--- a/docs/dev/src/connector/intro.md
+++ b/docs/dev/src/connector/intro.md
@@ -1,5 +1,8 @@
 # Develop Connectors
 
+This page describes the development workflow to develop connectors. For design docs, see
+- [Source](./source.md)
+
 RisingWave supports a lot of connectors (sources and sinks).
 However, developing connectors is tricky because it involves external systems:
 
diff --git a/docs/data-source.md b/docs/dev/src/connector/source.md
similarity index 87%
rename from docs/data-source.md
rename to docs/dev/src/connector/source.md
index 597f261390e14..302f688a5ed0f 100644
--- a/docs/data-source.md
+++ b/docs/dev/src/connector/source.md
@@ -1,20 +1,16 @@
-# Data Source
-
-- [Data Source](#data-source)
-  - [Components](#components)
-    - [Connectors](#connectors)
-    - [Enumerators](#enumerators)
-    - [ConnectorSource](#connectorsource)
-    - [SourceExecutor](#sourceexecutor)
-  - [How It Works](#how-it-works)
+# Source
 
 This page describes RisingWave's Data Source API and the architecture behind it. This may help if you are interested in how data sources work, or if you want to implement a new Data Source.
 
+For the workflow of developing connectors, see [Develop Connectors](../connector/intro.md).
+
+<!-- toc -->
+
 ## Components
 
 RisingWave's data source covers four parts: connectors, enumerators, ConnectorSource and SourceExecutor.
 
-![data source arch](../docs/images/data-source/data-source-arch.svg)
+![data source arch](../images/data-source/data-source-arch.svg)
 
 ### Connectors
 
@@ -41,7 +37,7 @@ pub trait SplitReader: Sized {
 
 ### Enumerators
 
-`Enumerator` periodically requests upstream to discover changes in splits, and in most cases the number of splits only increases. The enumerator is a separate task that runs on the [meta](./meta-service.md). If the upstream split changes, the enumerator notifies the connector by means of config change to change the subscription relationship.
+`Enumerator` periodically requests upstream to discover changes in splits, and in most cases the number of splits only increases. The enumerator is a separate task that runs on the [meta](../design/meta-service.md). If the upstream split changes, the enumerator notifies the connector by means of config change to change the subscription relationship.
 
 All enumerators need to implement the following trait.
 
diff --git a/docs/aggregation.md b/docs/dev/src/design/aggregation.md
similarity index 91%
rename from docs/aggregation.md
rename to docs/dev/src/design/aggregation.md
index 05d50c17967e2..bcb446a8ab73b 100644
--- a/docs/aggregation.md
+++ b/docs/dev/src/design/aggregation.md
@@ -13,7 +13,7 @@ TODO
 
 ## HashAggExecutor
 
-![aggregation components](./images/aggregation/agg-components.png)
+![aggregation components](../images/aggregation/agg-components.png)
 
 Within the `HashAggExecutor`, there are 4 main components:
 1. AggCalls.
@@ -40,10 +40,10 @@ For each of these aggregations, they have 1 state table (`AggStateStorage::Mater
 
 ### Initialization of `AggGroups`
 
-![init-agg-group](./images/aggregation/init-agg-group.png)
+![init-agg-group](../images/aggregation/init-agg-group.png)
 
 AggGroups are initialized when corresponding aggregation groups are not found in `AggGroupCache`.
 This could be either because the `AggGroupCache` got evicted,
 or its a new group key.
 
-It could take a while to initialize agg groups, hence we cache them in `AggGroupCache`.
\ No newline at end of file
+It could take a while to initialize agg groups, hence we cache them in `AggGroupCache`.
diff --git a/docs/architecture-design.md b/docs/dev/src/design/architecture-design.md
similarity index 95%
rename from docs/architecture-design.md
rename to docs/dev/src/design/architecture-design.md
index e89a945395050..1ec7e4de59635 100644
--- a/docs/architecture-design.md
+++ b/docs/dev/src/design/architecture-design.md
@@ -19,7 +19,7 @@ There are currently 4 types of nodes in the cluster:
    * **HummockManager**: Manages the SST file manifest and meta-info of Hummock storage.
    * **CompactionManager**: Manages the compaction status and task assignment of Hummock storage.
 
-![Architecture](./images/architecture-design/architecture.svg)
+![Architecture](../images/architecture-design/architecture.svg)
 
 The topmost component is the Postgres client. It issues queries through [TCP-based Postgres wire protocol](https://www.postgresql.org/docs/current/protocol.html).
 
@@ -41,13 +41,13 @@ Let's begin with a simple SELECT and see how it is executed.
 SELECT SUM(t.quantity) FROM t group by t.company;
 ```
 
-![Batch-Query](./images/architecture-design/batch-query.svg)
+![Batch-Query](../images/architecture-design/batch-query.svg)
 
 The query will be sliced into multiple *plan fragments*, each being an independent scheduling unit and probably with different parallelism. For simplicity, parallelism is usually set to the number of CPU cores in the cluster. For example, if there are 3 compute-nodes in the cluster, each with 4 CPU cores, then the parallelism will be set to 12 by default.
 
 Each parallel unit is called a *task*. Specifically, PlanFragment 2 will be distributed as 4 tasks to 4 CPU cores.
 
-![Plan-Fragments](./images/architecture-design/plan-fragments.svg)
+![Plan-Fragments](../images/architecture-design/plan-fragments.svg)
 
 Behind the TableScan operator, there's a storage engine called Hummock that stores the internal states, materialized views, and the tables. Note that only the materialized views and tables are queryable. The internal states are invisible to users.
 
@@ -62,7 +62,7 @@ For example:
 CREATE MATERIALIZED VIEW mv1 AS SELECT SUM(t.quantity) as q FROM t group by t.company;
 ```
 
-![Stream-Pipeline](./images/architecture-design/stream-pipeline.png)
+![Stream-Pipeline](../images/architecture-design/stream-pipeline.png)
 
 When the data source (Kafka, e.g.) propagates a bunch of records into the system, the materialized view will refresh automatically.
 
diff --git a/docs/backfill.md b/docs/dev/src/design/backfill.md
similarity index 97%
rename from docs/backfill.md
rename to docs/dev/src/design/backfill.md
index aac20615caf10..d23f7332b8d03 100644
--- a/docs/backfill.md
+++ b/docs/dev/src/design/backfill.md
@@ -221,7 +221,7 @@ Notice how `mv2` only needs the `id` column from `mv1`, and not the full `pk` wi
 
 #### Overview
 
-![backfill sides](./images/backfill/backfill-sides.png)
+![backfill sides](../images/backfill/backfill-sides.png)
 
 For `ArrangementBackfill`, we have 2 streams which we merge:
 upstream and historical streams.
@@ -230,7 +230,7 @@ to make sure `Barriers` can flow through the stream graph.
 Additionally, every epoch, we will refresh the historical stream, as upstream data gets checkpointed
 so our snapshot is stale.
 
-![polling](./images/backfill/polling.png)
+![polling](../images/backfill/polling.png)
 
 We will poll from this stream in backfill to get upstream and historical data chunks for processing,
 as well as barriers to checkpoint to backfill state.
@@ -239,7 +239,7 @@ For each chunk (DataChunk / StreamChunk), we may also need to do some further pr
 
 #### Schemas
 
-![schema](./images/backfill/schema.png)
+![schema](../images/backfill/schema.png)
 
 There are 3 schemas to consider when processing the backfill data:
 1. The state table schema of upstream.
@@ -258,7 +258,7 @@ to ensure the historical side and the upstream side have a consistent schema.
 
 #### Polling loop
 
-![handle_poll](./images/backfill/handle-poll.png)
+![handle_poll](../images/backfill/handle-poll.png)
 
 If we poll a chunk from the historical side, we will yield it to the downstream,
 and update the primary key (pk) we have backfilled to in the backfill state.
@@ -278,7 +278,7 @@ Then the polling loop will continue.
 
 ### Replication
 
-![replication_simple](./images/backfill/replication-simple.png)
+![replication_simple](../images/backfill/replication-simple.png)
 
 Previously, when doing snapshot reads to read **Historical Data**, backfill executor is able to read
 from the shared buffer for the previous epoch.
@@ -288,7 +288,7 @@ However, with `ArrangementBackfill`,
 we can't rely on the shared buffer of upstream,
 since it can be on a different parallel unit.
 
-![replication_replicated](./images/backfill/replication-replicated.png)
+![replication_replicated](../images/backfill/replication-replicated.png)
 
 So we need to make sure for the previous epoch, we buffer
 its updates somewhere to replicate the shared buffer.
@@ -314,7 +314,7 @@ to ensure the historical side and the upstream side have a consistent schema.
 Where (1) refers to the state table schema of upstream,
 and (2) refers to the output schema from upstream to arrangement backfill.
 
-![replication_example](./images/backfill/replication-example.png)
+![replication_example](../images/backfill/replication-example.png)
 
 Now let's consider an instance where (1) has the schema:
 
@@ -393,4 +393,4 @@ TODO
 
 ## Source Backfill
 
-TODO
\ No newline at end of file
+TODO
diff --git a/docs/batch-local-execution-mode.md b/docs/dev/src/design/batch-local-execution-mode.md
similarity index 94%
rename from docs/batch-local-execution-mode.md
rename to docs/dev/src/design/batch-local-execution-mode.md
index 219488acbcdb9..f60e59054acc8 100644
--- a/docs/batch-local-execution-mode.md
+++ b/docs/dev/src/design/batch-local-execution-mode.md
@@ -17,14 +17,14 @@ requirement of point queries, and in this article we introduce local execution m
 
 ## Design
 
-![Frontend Flow](./images/batch-local-execution-mode/frontend-flow.svg)
+![Frontend Flow](../images/batch-local-execution-mode/frontend-flow.svg)
 
 ## Example 1: select a from t where b in (1, 2, 3, 4)
 
 Let's use the above SQL as an example:
 
 
-![Example 1](./images/batch-local-execution-mode/example1.svg)
+![Example 1](../images/batch-local-execution-mode/example1.svg)
 
 The key changes from the distributed mode:
 
@@ -36,7 +36,7 @@ The key changes from the distributed mode:
 Following is the plan and execution of above sql in local mode:
 
 
-![Example 2](./images/batch-local-execution-mode/example2.svg)
+![Example 2](../images/batch-local-execution-mode/example2.svg)
 
 As explained above, the lookup join/exchange phase will be executed directly on frontend. The pushdown(filter/table, both the build and probe side) will be issued by executors rather than scheduler.
 
diff --git a/docs/checkpoint.md b/docs/dev/src/design/checkpoint.md
similarity index 97%
rename from docs/checkpoint.md
rename to docs/dev/src/design/checkpoint.md
index 4cba70c406b28..32125ce0af658 100644
--- a/docs/checkpoint.md
+++ b/docs/dev/src/design/checkpoint.md
@@ -20,7 +20,7 @@ The consistent checkpoints play 2 roles in our system.
 
 RisingWave makes checkpointing via [Chandy–Lamport algorithm](https://en.wikipedia.org/wiki/Chandy%E2%80%93Lamport_algorithm). A special kind of message, checkpoint barriers, is generated on streaming source and propagates across the streaming graph to the materialized views (or sink).
 
-![](./images/checkpoint/checkpoint.svg)
+![](../images/checkpoint/checkpoint.svg)
 
 To guarantee consistency, RisingWave introduces Chandy-Lamport algorithm as its checkpoint scheme.
 In particular, RisingWave periodically (every `barrier_interval_ms`) repeats the following procedure:
@@ -39,6 +39,6 @@ As is mentioned before, during checkpointing, every operator writes their change
 
 A local shared buffer is introduced to stage these uncommitted write batches. Once the checkpoint barriers have pass through all actors, the storage manager can notify all compute nodes to 'commit' their buffered write batches into the shared storage.
 
-![shared buffer](./images/checkpoint/shared-buffer.svg)
+![shared buffer](../images/checkpoint/shared-buffer.svg)
 
 Another benefit of shared buffer is that the write batches in a compute node can be compacted into a single SSTable file before uploading, which significantly reduces the number of SSTable files in Layer 0.
diff --git a/docs/consistent-hash.md b/docs/dev/src/design/consistent-hash.md
similarity index 92%
rename from docs/consistent-hash.md
rename to docs/dev/src/design/consistent-hash.md
index b3bdf1cf57559..43c3b973c04bc 100644
--- a/docs/consistent-hash.md
+++ b/docs/dev/src/design/consistent-hash.md
@@ -22,7 +22,7 @@ Here comes the main part, where we will construct a mapping that determines data
 
 For all data $k \in U_k$, where $U_k$ is an unbounded set, we apply a hash function $v = H(k)$, where $v$ falls to a limited range. The hash function $H$ ensures that all data are hashed **uniformly** to that range. We call $v$ vnode, namely virtual node, as is shown as the squares in the figure below.
 
-![initial data distribution](./images/consistent-hash/data-distribution.svg)
+![initial data distribution](../images/consistent-hash/data-distribution.svg)
 
 Then we have vnode mapping, which ensures that vnodes are mapped evenly to parallel units in the cluster. In other words, the number of vnodes that are mapped to each parallel unit should be as close as possible. This is denoted by different colors in the figure above. As is depicted, we have 3 parallel units (shown as circles), each taking $\frac{1}{3}$ of total vnodes. Vnode mapping is [constructed and maintained by meta](https://github.com/risingwavelabs/risingwave/blob/main/src/meta/src/stream/scheduler.rs).
 
@@ -34,17 +34,17 @@ Since $v = H(k)$, the way that data are mapped to vnodes will be invariant. Ther
 
 Let's take scaling out for example. Assume that we have one more parallel unit after scaling out, as is depicted as the orange circle in the figure below. Using the optimal strategy, we modify the vnode mapping in such a way that only $\frac{1}{4}$ of the data have to be moved, as is shown in the figure below. The vnodes whose data are required to be moved are highlighted with bold border in the figure.
 
-![optimal data redistribution](./images/consistent-hash/data-redistribution-1.svg)
+![optimal data redistribution](../images/consistent-hash/data-redistribution-1.svg)
 
 To minimize data movement when scaling occurs, we should be careful when we modify the vnode mapping. Below is an opposite example. Modifying vnode mapping like this will result in $\frac{1}{2}$ of the data being moved.
 
-![worst data redistribution](./images/consistent-hash/data-redistribution-2.svg)
+![worst data redistribution](../images/consistent-hash/data-redistribution-2.svg)
 
 ### Streaming
 
 We know that a fragment has several actors as its different parallelisms, and that upstream actors will send data to downstream actors via [dispatcher](./streaming-overview.md#actors). The figure below illustrates how actors distribute data based on consistent hash by example.
 
-![actor data distribution](./images/consistent-hash/actor-data.svg)
+![actor data distribution](../images/consistent-hash/actor-data.svg)
 
 In the figure, we can see that one upstream actor dispatches data to three downstream actors. The downstream actors are scheduled on the parallel units mentioned in previous example respectively.
 
@@ -78,15 +78,15 @@ We know that [Hummock](./state-store-overview.md#overview), our LSM-Tree-based s
 ```
 table_id | vnode | ...
 ```
-where `table_id` denotes the [state table](relational_table/storing-state-using-relational-table.md#relational-table-layer), and `vnode` is computed via $H$ on key of the data.
+where `table_id` denotes the [state table](relational-table.md), and `vnode` is computed via $H$ on key of the data.
 
 To illustrate this, let's revisit the [previous example](#streaming). Executors of an operator will share the same logical state table, just as is shown in the figure below:
 
-![actor state table](./images/consistent-hash/actor-state-table.svg)
+![actor state table](../images/consistent-hash/actor-state-table.svg)
 
 Now that we have 12 vnodes in total in the example, the data layout in storage will accordingly look like this:
 
-![storage data layout](./images/consistent-hash/storage-data-layout.svg)
+![storage data layout](../images/consistent-hash/storage-data-layout.svg)
 
 Note that we only show the logical sequence and aggregation of data in this illustration. The actual data may be separated into different SSTs in Hummock.
 
diff --git a/docs/data-model-and-encoding.md b/docs/dev/src/design/data-model-and-encoding.md
similarity index 86%
rename from docs/data-model-and-encoding.md
rename to docs/dev/src/design/data-model-and-encoding.md
index f27745a41be4d..c2a70c40909c6 100644
--- a/docs/data-model-and-encoding.md
+++ b/docs/dev/src/design/data-model-and-encoding.md
@@ -1,11 +1,6 @@
 # Data Model and Encoding
 
-- [Data Model and Encoding](#data-model-and-encoding)
-  - [Data Model](#data-model)
-  - [In-Memory Encoding](#in-memory-encoding)
-  - [On-Disk Encoding](#on-disk-encoding)
-
-<!-- Created by https://github.com/ekalinin/github-markdown-toc -->
+<!-- toc -->
 
 ## Data Model
 
@@ -26,7 +21,7 @@ Primitive data types:
 - Strings: `VARCHAR`
 - Temporals: `DATE`, `TIMESTAMP`, `TIMESTAMP WITH TIME ZONE`, `TIME`, `INTERVAL`
 
-Composite data types (WIP):
+Composite data types:
 
 - `Struct`: A structure with a list of named, strong-typed fields.
 - `List`: A variable-length list of values with same data type.
@@ -41,7 +36,7 @@ A Data Chunk consists of multiple columns and a visibility array, as is shown in
 
 A Stream Chunk consists of columns, visibility array and an additional `ops` column, as is shown in the right subgraph below. The `ops` column marks the operation of row, which can be one of `Delete`, `Insert`, `UpdateDelete` and `UpdateInsert`.
 
-![chunk](./images/data-model-and-encoding/chunk.svg)
+![chunk](../images/data-model-and-encoding/chunk.svg)
 
 ## On-Disk Encoding
 
@@ -49,9 +44,6 @@ A Stream Chunk consists of columns, visibility array and an additional `ops` col
 
 RisingWave stores user data in shared key-value storage called 'Hummock'. Tables, materialized views and checkpoints of internal streaming operators are encoded into key-value entries. Every field of a row, a.k.a. cell, is encoded as a key-value entry, except that `NULL` values are omitted.
 
-![row-format](./images/data-model-and-encoding/row-format.svg)
+![row-format](../images/data-model-and-encoding/row-format.svg)
 
 Considering that ordering matters in some cases, e.g. result set of an order-by query, fields of keys must preserve the order of original values after being encoded into bytes. This is what `memcomparable` is used for. For example, integers must be encoded in big-endian and the sign bit must be flipped to preserve order. In contrast, the encoding of values does not need to preserve order.
-
-
-
diff --git a/docs/keys.md b/docs/dev/src/design/keys.md
similarity index 99%
rename from docs/keys.md
rename to docs/dev/src/design/keys.md
index b0baa40c8716d..1e1bc056978d3 100644
--- a/docs/keys.md
+++ b/docs/dev/src/design/keys.md
@@ -62,4 +62,4 @@ Then when iterating over the keys from storage, the records are returned in the
 
 When the update stream comes, we can just use `id` to identify the records that need to be updated.
 We can get the whole record corresponding to the `id` and get the `i` from there.
-Then we can use that to update the materialized state accordingly.
\ No newline at end of file
+Then we can use that to update the materialized state accordingly.
diff --git a/docs/meta-service.md b/docs/dev/src/design/meta-service.md
similarity index 86%
rename from docs/meta-service.md
rename to docs/dev/src/design/meta-service.md
index cf10de6ff3bb4..4ec7b01d10e4f 100644
--- a/docs/meta-service.md
+++ b/docs/dev/src/design/meta-service.md
@@ -1,14 +1,6 @@
 # Meta Service
 
-- [Meta Service](#meta-service)
-  - [Background](#background)
-  - [Meta Store](#meta-store)
-  - [Types of Metadata](#types-of-metadata)
-    - [Catalog](#catalog)
-    - [Storage](#storage)
-  - [Push on Updates](#push-on-updates)
-
-<!-- Created by https://github.com/ekalinin/github-markdown-toc -->
+<!-- toc -->
 
 ## Background
 
@@ -16,7 +8,7 @@ RisingWave provides both real-time analytical query as well as high-concurrent a
 
 Meanwhile, components such as metadata provider, scheduler, monitoring are more suitable for a centralized design. For example, a typical on-premise deployment may look like below, where the dotted boxes represent minimal unit of deployment (VM or container).
 
-![Cluster Deployment](./images/meta-service/cluster-deployment.svg)
+![Cluster Deployment](../images/meta-service/cluster-deployment.svg)
 
 ## Meta Store
 
@@ -50,7 +42,6 @@ There are 2 choices on how to distribute information across multiple nodes.
 
 Currently, for simplicity, we choose the push-style approach for all kinds of metadata. This is implemented as `NotificationService` on meta service and `ObserverManager` on frontend and compute nodes.
 
-![Notification](./images/meta-service/notification.svg)
+![Notification](../images/meta-service/notification.svg)
 
 `ObserverManager` will register itself to meta service on bootstrap and subscribe metadata it needs. Afterwards, once metadata changed, the meta node streams the changes to it, expecting all subscribers to acknowledge.
-
diff --git a/docs/multi-object-store.md b/docs/dev/src/design/multi-object-store.md
similarity index 98%
rename from docs/multi-object-store.md
rename to docs/dev/src/design/multi-object-store.md
index fe12ce579adf3..699cd036f419d 100644
--- a/docs/multi-object-store.md
+++ b/docs/dev/src/design/multi-object-store.md
@@ -1,7 +1,6 @@
 # Build RisingWave with Multiple Object Storage Backends
 
-
-<!-- Created by https://github.com/ekalinin/github-markdown-toc -->
+<!-- toc -->
 
 ## Overview
 As a cloud-neutral database, RisingWave supports running on different (object) storage backends. Currently, these storage products include
@@ -76,4 +75,4 @@ After that, you need to [enable OpenDAL](https://github.com/risingwavelabs/risin
 
 You can also use WebHDFS as a lightweight alternative to HDFS. Hdfs is powered by HDFS's native java client. Users need to setup the hdfs services correctly. But webhdfs can access from HTTP API and no extra setup needed. The way to start WebHDFS is basically the same as hdfs, but its default name_node is `127.0.0.1:9870`.
 
-Once these configurations are set, run `./risedev d hdfs` or `./risedev d webhdfs`, then you can run RisingWave on HDFS(WebHDFS).
\ No newline at end of file
+Once these configurations are set, run `./risedev d hdfs` or `./risedev d webhdfs`, then you can run RisingWave on HDFS(WebHDFS).
diff --git a/docs/mv-on-mv.md b/docs/dev/src/design/mv-on-mv.md
similarity index 95%
rename from docs/mv-on-mv.md
rename to docs/dev/src/design/mv-on-mv.md
index e66b48224f43f..d1091c0c0ac82 100644
--- a/docs/mv-on-mv.md
+++ b/docs/dev/src/design/mv-on-mv.md
@@ -1,4 +1,4 @@
-# Create MView on Top of MView
+# MView on Top of MView
 
 ## Background
 
@@ -19,7 +19,7 @@ create materialized view mv3 as select count(v1) as count_v1 from mv1;
 
 In physical representation, we introduce a dispatcher operator type, *Broadcast*. Broadcast dispatcher, as its name indicates, will dispatch every message to multiple downstreams. To simplify our design, we can assume that every MViewOperator has a `Broadcast` output, with zero or more downstreams.
 
-![fig1](../docs/images/mv-on-mv/mv-on-mv-01.svg)
+![fig1](../images/mv-on-mv/mv-on-mv-01.svg)
 
 ### Create new mview online
 
@@ -27,7 +27,7 @@ Assume that we already have a materialized view mv1, and we want to create a new
 
 The Chain operator has two inputs. The first one will be a batch query, denoted by the blue patterns in the figure below, which is a finite append-only stream (the snapshot of historical data in the base mview). The second one is its original input, an infinite stream, denoted by the red patterns.
 
-![fig2](../docs/images/mv-on-mv/mv-on-mv-02.svg)
+![fig2](../images/mv-on-mv/mv-on-mv-02.svg)
 
 The full process of creation is:
 
diff --git a/docs/relational_table/storing-state-using-relational-table.md b/docs/dev/src/design/relational-table.md
similarity index 64%
rename from docs/relational_table/storing-state-using-relational-table.md
rename to docs/dev/src/design/relational-table.md
index f289b83507b16..f49907726e20e 100644
--- a/docs/relational_table/storing-state-using-relational-table.md
+++ b/docs/dev/src/design/relational-table.md
@@ -1,14 +1,6 @@
 # Storing State Using Relational Table
 
-- [Storing State Using Relational Table](#storing-state-using-relational-table)
-  - [Row-based Encoding](#row-based-encoding)
-  - [Relational Table Layer](#relational-table-layer)
-    - [Write Path](#write-path)
-    - [Read Path](#read-path)
-
-
-
-<!-- Created by https://github.com/ekalinin/github-markdown-toc -->
+<!-- toc -->
 
 ## Row-based Encoding
 
@@ -23,8 +15,6 @@ We implement a relational table layer as the bridge between executors and KV sta
 | join     | table_id \| join_key \| pk | materialized value |
 | agg | table_id \| group_key | agg_value |
 
-For the detailed schema, please check [doc](relational-table-schema.md)
-
 <!-- Todo: link cconsistence hash doc and state table agg doc -->
 ## Relational Table Layer
 [source code](https://github.com/risingwavelabs/risingwave/blob/4e66ca3d41435c64af26b5e0003258c4f7116822/src/storage/src/table/state_table.rs)
@@ -36,13 +26,13 @@ Relational table layer consists of State Table, Mem Table and Storage Table. The
 State Table provides the table operations by these APIs: `get_row`, `scan`, `insert_row`, `delete_row` and `update_row`, which are the read and write interfaces for streaming executors. The Mem Table is an in-memory buffer for caching table operations during one epoch. The Storage Table is read only, and will output the partial columns  upper level needs.
 
 
-![Overview of Relational Table](../images/relational-table-layer/relational-table-01.svg)
+![Overview of Relational Table](../images/relational-table/relational-table-01.svg)
 ### Write Path
 To write into KV state store, executors first perform operations on State Table, and these operations will be cached in Mem Table. Once a barrier flows through one executor, executor will flush the cached operations into state store. At this moment, State Table will covert these operations into kv pairs and write to state store with specific epoch.
 
 For example, an executor performs `insert(a, b, c)` and `delete(d, e, f)` through the State Table APIs, Mem Table first caches these two operations in memory. After receiving new barrier, State Table converts these two operations into KV operations by row-based format, and writes these KV operations into state store (Hummock).
 
-![write example](../images/relational-table-layer/relational-table-03.svg)
+![write example](../images/relational-table/relational-table-03.svg)
 ### Read Path
 In streaming mode, executors should be able to read the latest written data, which means uncommitted data is visible. The data in Mem Table (memory) is fresher than that in shared storage (state store). State Table provides both point-get and scan to read from state store by merging data from Mem Table and Storage Table.
 #### Get
@@ -68,4 +58,36 @@ Get(pk = 3): [3, 3333, 3333]
 #### Scan
 Scan on relational table is implemented by `StateTableIter`, which is a merge iterator of `MemTableIter` and `StorageIter`. If a pk exists in both KV state store (shared storage) and memory (MemTable), result of `MemTableIter` is returned. For example, in the  following figure, `StateTableIter` will generate `1->4->5->6` in order.
 
-![Scan example](../images/relational-table-layer/relational-table-02.svg)
+![Scan example](../images/relational-table/relational-table-02.svg)
+
+
+## Example: HashAgg
+
+In this doc, we will take HashAgg with extreme state (`max`, `min`) or value state (`sum`, `count`) for example, and introduce a more detailed design for the internal table schema.
+
+[Code](https://github.com/risingwavelabs/risingwave/blob/7f9ad2240712aa0cfe3edffb4535d43b42f32cc5/src/frontend/src/optimizer/plan_node/logical_agg.rs#L144)
+
+### Table id
+`table_id` is a globally unique id allocated in meta for each relational table object. Meta is responsible for traversing the Plan Tree and calculating the total number of Relational Tables needed. For example, the Hash Join Operator needs 2, one for the left table and one for the right table. The number of tables needed for Agg depends on the number of agg calls.
+
+### Value State (Sum, Count)
+Query example:
+```sql
+select sum(v2), count(v3) from t group by v1
+```
+
+This query will need to initiate 2 Relational Tables. The schema is `table_id/group_key`.
+
+### Extreme State (Max, Min)
+Query example:
+```sql
+select max(v2), min(v3) from t group by v1
+```
+
+This query will need to initiate 2 Relational Tables. If the upstream is not append-only, the schema becomes `table_id/group_key/sort_key/upstream_pk`.
+
+The order of `sort_key` depends on the agg call kind. For example, if it's `max()`, `sort_key` will order with `Ascending`. if it's `min()`, `sort_key` will order with `Descending`.
+The `upstream_pk` is also appended to ensure the uniqueness of the key.
+This design allows the streaming executor not to read all the data from the storage when the cache fails, but only a part of it. The streaming executor will try to write all streaming data to storage, because there may be `update` or `delete` operations in the stream, it's impossible to always guarantee correct results without storing all data.
+
+If `t` is created with append-only flag, the schema becomes `table_id/group_key`, which is the same for Value State. This is because in the append-only mode, there is no `update` or `delete` operation, so the cache will never miss. Therefore, we only need to write one value to the storage.
diff --git a/docs/shared-buffer.md b/docs/dev/src/design/shared-buffer.md
similarity index 95%
rename from docs/shared-buffer.md
rename to docs/dev/src/design/shared-buffer.md
index 2b63a040b4c9a..846d8bd1d064a 100644
--- a/docs/shared-buffer.md
+++ b/docs/dev/src/design/shared-buffer.md
@@ -1,14 +1,6 @@
 # The Hummock Shared Buffer
 
-Table of contents:
-
-- [Introduction](#introduction)
-- [Part 1: Async Checkpoint](#part-1-async-checkpoint)
-  - [Write Path](#write-path)
-  - [Read Path](#read-path)
-- [Part 2: Write Anytime / Async Flush](#part-2-write-anytime--async-flush)
-  - [A New Merge Iterator](#a-new-merge-iterator)
-    - [Considerations](#considerations)
+<!-- toc -->
 
 ## Introduction
 
@@ -137,4 +129,4 @@ For all data a, b of the same type, we must ensure that:
 ```
 in-memory representation of a < in-memory representation of b,
 iff memcomparable(a) < memcomparable(b)
-```
\ No newline at end of file
+```
diff --git a/docs/state-store-overview.md b/docs/dev/src/design/state-store-overview.md
similarity index 92%
rename from docs/state-store-overview.md
rename to docs/dev/src/design/state-store-overview.md
index 0fc64516ac52f..a11d9528d2586 100644
--- a/docs/state-store-overview.md
+++ b/docs/dev/src/design/state-store-overview.md
@@ -1,18 +1,6 @@
 # An Overview of RisingWave State Store
 
-- [An Overview of RisingWave State Store](#an-overview-of-risingwave-state-store)
-  - [Overview](#overview)
-  - [Architecture](#architecture)
-  - [The Hummock User API](#the-hummock-user-api)
-  - [Hummock Internals](#hummock-internals)
-    - [Storage Format](#storage-format)
-    - [Write Path](#write-path)
-    - [Read Path](#read-path)
-    - [Compaction](#compaction)
-    - [Transaction Management with Hummock Manager](#transaction-management-with-hummock-manager)
-    - [Checkpointing in Streaming](#checkpointing-in-streaming)
-
-<!-- Created by https://github.com/ekalinin/github-markdown-toc -->
+<!-- toc -->
 
 ## Overview
 
@@ -22,7 +10,7 @@ In RisingWave, all streaming executors store their data into a state store. This
 
 Reading this document requires prior knowledge of LSM-Tree-based KV storage engines, like RocksDB, LevelDB, etc.
 
-![Overview of Architecture](images/state-store-overview/state-store-overview-01.svg)
+![Overview of Architecture](../images/state-store-overview/state-store-overview-01.svg)
 
 Hummock consists of a manager service on the meta node, clients on worker nodes (including compute nodes, frontend nodes, and compactor nodes), and a shared storage to store files (SSTs). Every time a new write batch is produced, the Hummock client will upload those files to shared storage, and notify the Hummock manager of the new data. With compaction going on, new files will be added and unused files will be vacuumed. The Hummock manager will take care of the lifecycle of a file — is a file being used? can we delete a file? etc.
 
@@ -104,7 +92,7 @@ The Hummock client will batch writes and generate SSTs to sync to the underlying
 After the SST is uploaded to an S3-compatible service, the Hummock client will let the Hummock manager know there's a new table.
 The list of all SSTs along with some metadata forms a ***version***. When the Hummock client adds new SSTs to the Hummock manager, a new version will be generated with the new set of SST files.
 
-![Write Path](images/state-store-overview/state-store-overview-02.svg)
+![Write Path](../images/state-store-overview/state-store-overview-02.svg)
 
 ### Read Path
 
@@ -114,7 +102,7 @@ For every read operation (`scan`, `get`), we will first select SSTs that might c
 
 For `scan`, we simply select by overlapping key range. For point get, we will filter SSTs further by Bloom filter. After that, we will compose a single `MergeIterator` over all SSTs. The `MergeIterator` will return all keys in range along with their epochs. Then, we will create `UserIterator` over `MergeIterator`, and for all user keys, the user iterator will pick the first full key whose epoch <= read epoch. Therefore, users can perform a snapshot read from Hummock based on the given epoch. The snapshot should be acquired beforehand and released afterwards.
 
-![Read Path](images/state-store-overview/state-store-overview-03.svg)
+![Read Path](../images/state-store-overview/state-store-overview-03.svg)
 
 Hummock implements the following iterators:
 - `BlockIterator`: iterates a block of an SSTable.
@@ -148,7 +136,7 @@ As mentioned in [Read Path](#read-path), reads are performed on a ***version***
 
 The SQL frontend will get the latest epoch from the meta service. Then, it will embed the epoch number into SQL plans, so that all compute nodes will read from that epoch. In theory, both SQL frontend and compute nodes will ***pin the snapshot***, to handle the case that frontend goes down and the compute nodes are still reading from Hummock (#622). However, to simplify the process, currently we ***only pin on the frontend side***.
 
-![Hummock Service](images/state-store-overview/state-store-overview-04.svg)
+![Hummock Service](../images/state-store-overview/state-store-overview-04.svg)
 
 Hummock only guarantees that writes on one node can be immediately read from the same node. However, the worker nodes running batch queries might have a slightly outdated version when a batch query plan is received (due to the local version caching). Therefore, we have a `wait_epoch` interface to wait until the local cached version contains full data of one epoch.
 
@@ -164,7 +152,7 @@ From the perspective of the streaming executors, when they receive a barrier, th
 
 Here we have two cases: Agg executors always persist and produce new write batches when receiving a barrier; Join executors (in the future when async flush gets implemented) will produce write batches within an epoch.
 
-![Checkpoint in Streaming](images/state-store-overview/state-store-overview-05.svg)
+![Checkpoint in Streaming](../images/state-store-overview/state-store-overview-05.svg)
 
 Streaming executors cannot control when data will be persisted — they can only write to Hummock's `shared buffer`. When a barrier flows across the system and is collected by the meta service, we can ensure that all executors have written their states of ***the previous epoch*** to the shared buffer, so we can initiate checkpoint process on all worker nodes, and upload SSTs to persistent remote storage.
 
diff --git a/docs/streaming-overview.md b/docs/dev/src/design/streaming-overview.md
similarity index 91%
rename from docs/streaming-overview.md
rename to docs/dev/src/design/streaming-overview.md
index 2379fe2db13d3..d4a80c2a66c48 100644
--- a/docs/streaming-overview.md
+++ b/docs/dev/src/design/streaming-overview.md
@@ -1,16 +1,6 @@
 # An Overview of the RisingWave Streaming Engine
 
-- [An Overview of the RisingWave Streaming Engine](#an-overview-of-risingwave-streaming-engine)
-  - [Overview](#overview)
-  - [Architecture](#architecture)
-  - [Actors, executors, and states](#actors-executors-and-states)
-    - [Actors](#actors)
-    - [Executors](#executors)
-  - [Checkpoint, Consistency, and Fault tolerance](#checkpoint-consistency-and-fault-tolerance)
-    - [Barrier based checkpoint](#barrier-based-checkpoint)
-    - [Fault tolerance](#fault-tolerance)
-
-<!-- Created by https://github.com/ekalinin/github-markdown-toc -->
+<!-- toc -->
 
 ## Overview
 
@@ -26,7 +16,7 @@ In this document we give an overview of the RisingWave streaming engine.
 
 ## Architecture
 
-![streaming-architecture](./images/streaming-overview/streaming-architecture.svg)
+![streaming-architecture](../images/streaming-overview/streaming-architecture.svg)
 
 The overall architecture of RisingWave is depicted in the figure above. In brief, RisingWave streaming engine consists of three sets of nodes: frontend, compute nodes, and meta service. The frontend node consists of the serving layer, handling users' SQL requests concurrently. Underlying is the processing layer. Each compute node hosts a collection of long-running actors for stream processing. All actors access a shared persistence layer of storage (currently AWS S3) as its state storage. The meta service maintains all meta-information and coordinates the whole cluster.
 
@@ -38,7 +28,7 @@ When receiving a create materialized view statement at the frontend, a materiali
 4. Initializing the job at the backend. The meta service notifies all compute nodes to start serving streaming pipelines.
 ## Actors, executors, and states
 
-![streaming-executor](./images/streaming-overview/streaming-executor-and-compute-node.svg)
+![streaming-executor](../images/streaming-overview/streaming-executor-and-compute-node.svg)
 
 ### Actors
 
@@ -75,4 +65,3 @@ See more detailed descriptions on [Checkpoint](./checkpoint.md).
 ### Fault tolerance
 
 When the streaming engine crashes down, the system must globally rollback to a previous consistent snapshot. To achieve this, whenever the meta detects the failover of some certain compute node or any undergoing checkpoint procedure, it triggers a recovery process. After rebuilding the streaming pipeline, each executor will reset its local state from a consistent snapshot on the storage and recover its computation.
-
diff --git a/docs/images/aggregation/agg-components.png b/docs/dev/src/images/aggregation/agg-components.png
similarity index 100%
rename from docs/images/aggregation/agg-components.png
rename to docs/dev/src/images/aggregation/agg-components.png
diff --git a/docs/images/aggregation/init-agg-group.png b/docs/dev/src/images/aggregation/init-agg-group.png
similarity index 100%
rename from docs/images/aggregation/init-agg-group.png
rename to docs/dev/src/images/aggregation/init-agg-group.png
diff --git a/docs/images/architecture-design/architecture.svg b/docs/dev/src/images/architecture-design/architecture.svg
similarity index 100%
rename from docs/images/architecture-design/architecture.svg
rename to docs/dev/src/images/architecture-design/architecture.svg
diff --git a/docs/images/architecture-design/batch-query.svg b/docs/dev/src/images/architecture-design/batch-query.svg
similarity index 100%
rename from docs/images/architecture-design/batch-query.svg
rename to docs/dev/src/images/architecture-design/batch-query.svg
diff --git a/docs/images/architecture-design/plan-fragments.svg b/docs/dev/src/images/architecture-design/plan-fragments.svg
similarity index 100%
rename from docs/images/architecture-design/plan-fragments.svg
rename to docs/dev/src/images/architecture-design/plan-fragments.svg
diff --git a/docs/images/architecture-design/stream-pipeline.png b/docs/dev/src/images/architecture-design/stream-pipeline.png
similarity index 100%
rename from docs/images/architecture-design/stream-pipeline.png
rename to docs/dev/src/images/architecture-design/stream-pipeline.png
diff --git a/docs/images/backfill/backfill-sides.png b/docs/dev/src/images/backfill/backfill-sides.png
similarity index 100%
rename from docs/images/backfill/backfill-sides.png
rename to docs/dev/src/images/backfill/backfill-sides.png
diff --git a/docs/images/backfill/handle-poll.png b/docs/dev/src/images/backfill/handle-poll.png
similarity index 100%
rename from docs/images/backfill/handle-poll.png
rename to docs/dev/src/images/backfill/handle-poll.png
diff --git a/docs/images/backfill/polling.png b/docs/dev/src/images/backfill/polling.png
similarity index 100%
rename from docs/images/backfill/polling.png
rename to docs/dev/src/images/backfill/polling.png
diff --git a/docs/images/backfill/replication-example.png b/docs/dev/src/images/backfill/replication-example.png
similarity index 100%
rename from docs/images/backfill/replication-example.png
rename to docs/dev/src/images/backfill/replication-example.png
diff --git a/docs/images/backfill/replication-replicated.png b/docs/dev/src/images/backfill/replication-replicated.png
similarity index 100%
rename from docs/images/backfill/replication-replicated.png
rename to docs/dev/src/images/backfill/replication-replicated.png
diff --git a/docs/images/backfill/replication-simple.png b/docs/dev/src/images/backfill/replication-simple.png
similarity index 100%
rename from docs/images/backfill/replication-simple.png
rename to docs/dev/src/images/backfill/replication-simple.png
diff --git a/docs/images/backfill/schema.png b/docs/dev/src/images/backfill/schema.png
similarity index 100%
rename from docs/images/backfill/schema.png
rename to docs/dev/src/images/backfill/schema.png
diff --git a/docs/images/batch-local-execution-mode/example1.svg b/docs/dev/src/images/batch-local-execution-mode/example1.svg
similarity index 100%
rename from docs/images/batch-local-execution-mode/example1.svg
rename to docs/dev/src/images/batch-local-execution-mode/example1.svg
diff --git a/docs/images/batch-local-execution-mode/example2.svg b/docs/dev/src/images/batch-local-execution-mode/example2.svg
similarity index 100%
rename from docs/images/batch-local-execution-mode/example2.svg
rename to docs/dev/src/images/batch-local-execution-mode/example2.svg
diff --git a/docs/images/batch-local-execution-mode/frontend-flow.svg b/docs/dev/src/images/batch-local-execution-mode/frontend-flow.svg
similarity index 100%
rename from docs/images/batch-local-execution-mode/frontend-flow.svg
rename to docs/dev/src/images/batch-local-execution-mode/frontend-flow.svg
diff --git a/docs/images/checkpoint/checkpoint.svg b/docs/dev/src/images/checkpoint/checkpoint.svg
similarity index 100%
rename from docs/images/checkpoint/checkpoint.svg
rename to docs/dev/src/images/checkpoint/checkpoint.svg
diff --git a/docs/images/checkpoint/shared-buffer.svg b/docs/dev/src/images/checkpoint/shared-buffer.svg
similarity index 100%
rename from docs/images/checkpoint/shared-buffer.svg
rename to docs/dev/src/images/checkpoint/shared-buffer.svg
diff --git a/docs/images/consistent-hash/actor-data.svg b/docs/dev/src/images/consistent-hash/actor-data.svg
similarity index 100%
rename from docs/images/consistent-hash/actor-data.svg
rename to docs/dev/src/images/consistent-hash/actor-data.svg
diff --git a/docs/images/consistent-hash/actor-state-table.svg b/docs/dev/src/images/consistent-hash/actor-state-table.svg
similarity index 100%
rename from docs/images/consistent-hash/actor-state-table.svg
rename to docs/dev/src/images/consistent-hash/actor-state-table.svg
diff --git a/docs/images/consistent-hash/data-distribution.svg b/docs/dev/src/images/consistent-hash/data-distribution.svg
similarity index 100%
rename from docs/images/consistent-hash/data-distribution.svg
rename to docs/dev/src/images/consistent-hash/data-distribution.svg
diff --git a/docs/images/consistent-hash/data-redistribution-1.svg b/docs/dev/src/images/consistent-hash/data-redistribution-1.svg
similarity index 100%
rename from docs/images/consistent-hash/data-redistribution-1.svg
rename to docs/dev/src/images/consistent-hash/data-redistribution-1.svg
diff --git a/docs/images/consistent-hash/data-redistribution-2.svg b/docs/dev/src/images/consistent-hash/data-redistribution-2.svg
similarity index 100%
rename from docs/images/consistent-hash/data-redistribution-2.svg
rename to docs/dev/src/images/consistent-hash/data-redistribution-2.svg
diff --git a/docs/images/consistent-hash/storage-data-layout.svg b/docs/dev/src/images/consistent-hash/storage-data-layout.svg
similarity index 100%
rename from docs/images/consistent-hash/storage-data-layout.svg
rename to docs/dev/src/images/consistent-hash/storage-data-layout.svg
diff --git a/docs/images/data-model-and-encoding/chunk.svg b/docs/dev/src/images/data-model-and-encoding/chunk.svg
similarity index 100%
rename from docs/images/data-model-and-encoding/chunk.svg
rename to docs/dev/src/images/data-model-and-encoding/chunk.svg
diff --git a/docs/images/data-model-and-encoding/row-format.svg b/docs/dev/src/images/data-model-and-encoding/row-format.svg
similarity index 100%
rename from docs/images/data-model-and-encoding/row-format.svg
rename to docs/dev/src/images/data-model-and-encoding/row-format.svg
diff --git a/docs/images/data-source/data-source-arch.svg b/docs/dev/src/images/data-source/data-source-arch.svg
similarity index 100%
rename from docs/images/data-source/data-source-arch.svg
rename to docs/dev/src/images/data-source/data-source-arch.svg
diff --git a/docs/images/logo-title.svg b/docs/dev/src/images/logo-title.svg
similarity index 100%
rename from docs/images/logo-title.svg
rename to docs/dev/src/images/logo-title.svg
diff --git a/docs/images/logo.svg b/docs/dev/src/images/logo.svg
similarity index 100%
rename from docs/images/logo.svg
rename to docs/dev/src/images/logo.svg
diff --git a/docs/images/meta-service/cluster-deployment.svg b/docs/dev/src/images/meta-service/cluster-deployment.svg
similarity index 100%
rename from docs/images/meta-service/cluster-deployment.svg
rename to docs/dev/src/images/meta-service/cluster-deployment.svg
diff --git a/docs/images/meta-service/notification.svg b/docs/dev/src/images/meta-service/notification.svg
similarity index 100%
rename from docs/images/meta-service/notification.svg
rename to docs/dev/src/images/meta-service/notification.svg
diff --git a/docs/images/mv-on-mv/mv-on-mv-01.svg b/docs/dev/src/images/mv-on-mv/mv-on-mv-01.svg
similarity index 100%
rename from docs/images/mv-on-mv/mv-on-mv-01.svg
rename to docs/dev/src/images/mv-on-mv/mv-on-mv-01.svg
diff --git a/docs/images/mv-on-mv/mv-on-mv-02.svg b/docs/dev/src/images/mv-on-mv/mv-on-mv-02.svg
similarity index 100%
rename from docs/images/mv-on-mv/mv-on-mv-02.svg
rename to docs/dev/src/images/mv-on-mv/mv-on-mv-02.svg
diff --git a/docs/images/relational-table-layer/relational-table-01.svg b/docs/dev/src/images/relational-table/relational-table-01.svg
similarity index 100%
rename from docs/images/relational-table-layer/relational-table-01.svg
rename to docs/dev/src/images/relational-table/relational-table-01.svg
diff --git a/docs/images/relational-table-layer/relational-table-02.svg b/docs/dev/src/images/relational-table/relational-table-02.svg
similarity index 100%
rename from docs/images/relational-table-layer/relational-table-02.svg
rename to docs/dev/src/images/relational-table/relational-table-02.svg
diff --git a/docs/images/relational-table-layer/relational-table-03.svg b/docs/dev/src/images/relational-table/relational-table-03.svg
similarity index 100%
rename from docs/images/relational-table-layer/relational-table-03.svg
rename to docs/dev/src/images/relational-table/relational-table-03.svg
diff --git a/docs/images/state-store-overview/state-store-overview-01.svg b/docs/dev/src/images/state-store-overview/state-store-overview-01.svg
similarity index 100%
rename from docs/images/state-store-overview/state-store-overview-01.svg
rename to docs/dev/src/images/state-store-overview/state-store-overview-01.svg
diff --git a/docs/images/state-store-overview/state-store-overview-02.svg b/docs/dev/src/images/state-store-overview/state-store-overview-02.svg
similarity index 100%
rename from docs/images/state-store-overview/state-store-overview-02.svg
rename to docs/dev/src/images/state-store-overview/state-store-overview-02.svg
diff --git a/docs/images/state-store-overview/state-store-overview-03.svg b/docs/dev/src/images/state-store-overview/state-store-overview-03.svg
similarity index 100%
rename from docs/images/state-store-overview/state-store-overview-03.svg
rename to docs/dev/src/images/state-store-overview/state-store-overview-03.svg
diff --git a/docs/images/state-store-overview/state-store-overview-04.svg b/docs/dev/src/images/state-store-overview/state-store-overview-04.svg
similarity index 100%
rename from docs/images/state-store-overview/state-store-overview-04.svg
rename to docs/dev/src/images/state-store-overview/state-store-overview-04.svg
diff --git a/docs/images/state-store-overview/state-store-overview-05.svg b/docs/dev/src/images/state-store-overview/state-store-overview-05.svg
similarity index 100%
rename from docs/images/state-store-overview/state-store-overview-05.svg
rename to docs/dev/src/images/state-store-overview/state-store-overview-05.svg
diff --git a/docs/images/streaming-overview/streaming-architecture.svg b/docs/dev/src/images/streaming-overview/streaming-architecture.svg
similarity index 100%
rename from docs/images/streaming-overview/streaming-architecture.svg
rename to docs/dev/src/images/streaming-overview/streaming-architecture.svg
diff --git a/docs/images/streaming-overview/streaming-executor-and-compute-node.svg b/docs/dev/src/images/streaming-overview/streaming-executor-and-compute-node.svg
similarity index 100%
rename from docs/images/streaming-overview/streaming-executor-and-compute-node.svg
rename to docs/dev/src/images/streaming-overview/streaming-executor-and-compute-node.svg
diff --git a/docs/dev/src/intro.md b/docs/dev/src/intro.md
index c4317c8d71dd3..011ad4a149f83 100644
--- a/docs/dev/src/intro.md
+++ b/docs/dev/src/intro.md
@@ -1,6 +1,6 @@
 # Introduction
 
-This guide is intended to be used by contributors to learn about how to develop RisingWave. The instructions about how to submit code changes are included in [contributing guidelines](./contributing.md).
+This guide is intended to be used by contributors to learn about how to develop RisingWave. The instructions about how to submit code changes are included in [contribution guidelines](./contributing.md).
 
 If you have questions, you can search for existing discussions or start a new discussion in the [Discussions forum of RisingWave](https://github.com/risingwavelabs/risingwave/discussions), or ask in the RisingWave Community channel on Slack. Please use the [invitation link](https://risingwave.com/slack) to join the channel.
 
diff --git a/docs/metrics.md b/docs/dev/src/metrics.md
similarity index 98%
rename from docs/metrics.md
rename to docs/dev/src/metrics.md
index b0216c07fc83e..14d98c7a365ea 100644
--- a/docs/metrics.md
+++ b/docs/dev/src/metrics.md
@@ -5,7 +5,7 @@ It covers what each metric measures, and what information we may derive from it.
 
 ## Barrier Latency
 
-Prerequisite: [Checkpoint](./checkpoint.md)
+Prerequisite: [Checkpoint](./design/checkpoint.md)
 
 This metric measures the duration from which a barrier is injected into **all** sources in the stream graph,
 to the barrier flown through all executors in the graph.
diff --git a/docs/dev/src/tests/intro.md b/docs/dev/src/tests/intro.md
index 8db7185b44eab..7e99ca9b20884 100644
--- a/docs/dev/src/tests/intro.md
+++ b/docs/dev/src/tests/intro.md
@@ -18,7 +18,7 @@ RisingWave requires all code to pass fmt, clippy, sort and hakari checks. Run th
 
 There are also some miscellaneous checks. See `ci/scripts/misc-check.sh`.
 
-## Unit tests
+### Unit and integration tests
 
 RiseDev runs unit tests with cargo-nextest. To run unit tests:
 
@@ -26,6 +26,13 @@ RiseDev runs unit tests with cargo-nextest. To run unit tests:
 ./risedev test          # Run unit tests
 ```
 
+Some ideas and caveats for writing tests:
+- Use [expect_test](https://github.com/rust-analyzer/expect-test) to write data driven tests that can automatically update results.
+- It's recommended to write new tests as *integration tests* (i.e. in `tests/` directory) instead of *unit tests* (i.e. in `src/` directory).
+
+  Besides, put integration tests under `tests/integration_tests/*.rs`, instead of `tests/*.rs`. See [Delete Cargo Integration Tests](https://matklad.github.io/2021/02/27/delete-cargo-integration-tests.html) and [#9878](https://github.com/risingwavelabs/risingwave/issues/9878), for more details.
+
+You might want to read [How to Test](https://matklad.github.io/2021/05/31/how-to-test.html) for more good ideas on testing.
 
 ## Planner tests
 
diff --git a/docs/relational_table/relational-table-schema.md b/docs/relational_table/relational-table-schema.md
deleted file mode 100644
index 64cd615feda25..0000000000000
--- a/docs/relational_table/relational-table-schema.md
+++ /dev/null
@@ -1,35 +0,0 @@
-# Relational Table Schema
-
-We introduce the rough row-based encoding format in [relational states](storing-state-using-relational-table.md#row-based-encoding)
-
-In this doc, we will take HashAgg with extreme state (`max`, `min`) or value state (`sum`, `count`) for example, and introduce a more detailed design for the internal table schema.
-
-[Code](https://github.com/risingwavelabs/risingwave/blob/7f9ad2240712aa0cfe3edffb4535d43b42f32cc5/src/frontend/src/optimizer/plan_node/logical_agg.rs#L144)
-
-## Table id
-`table_id` is a globally unique id allocated in meta for each relational table object. Meta is responsible for traversing the Plan Tree and calculating the total number of Relational Tables needed. For example, the Hash Join Operator needs 2, one for the left table and one for the right table. The number of tables needed for Agg depends on the number of agg calls.
-
-## Value State (Sum, Count)
-Query example:
-```sql
-select sum(v2), count(v3) from t group by v1
-```
-
-This query will need to initiate 2 Relational Tables. The schema is `table_id/group_key`.
-
-## Extreme State (Max, Min)
-Query example:
-```sql
-select max(v2), min(v3) from t group by v1
-```
-
-This query will need to initiate 2 Relational Tables. If the upstream is not append-only, the schema becomes `table_id/group_key/sort_key/upstream_pk`.
-
-The order of `sort_key` depends on the agg call kind. For example, if it's `max()`, `sort_key` will order with `Ascending`. if it's `min()`, `sort_key` will order with `Descending`.
-The `upstream_pk` is also appended to ensure the uniqueness of the key.
-This design allows the streaming executor not to read all the data from the storage when the cache fails, but only a part of it. The streaming executor will try to write all streaming data to storage, because there may be `update` or `delete` operations in the stream, it's impossible to always guarantee correct results without storing all data.
-
-If `t` is created with append-only flag, the schema becomes `table_id/group_key`, which is the same for Value State. This is because in the append-only mode, there is no `update` or `delete` operation, so the cache will never miss. Therefore, we only need to write one value to the storage.
-
-
-
diff --git a/docs/rustdoc/README.md b/docs/rustdoc/README.md
index 1b3e70e1113c2..0adf956748290 100644
--- a/docs/rustdoc/README.md
+++ b/docs/rustdoc/README.md
@@ -1,6 +1,6 @@
 This folder contains files for generating a nice rustdoc index page.
 
-Online version (for latest main): <https://risingwavelabs.github.io/risingwave/>
+Online version (for latest main): <https://risingwavelabs.github.io/risingwave/rustdoc>
 
 To build and open locally, run the following command in the project root:
 
diff --git a/docs/rustdoc/index.md b/docs/rustdoc/index.md
index cfb74b8055b8a..a76edb23cb2b4 100644
--- a/docs/rustdoc/index.md
+++ b/docs/rustdoc/index.md
@@ -4,9 +4,7 @@ Welcome to an overview of the developer documentations of RisingWave!
 
 ## Developer Docs
 
-To learn how to develop RisingWave, see the [RisingWave Developer Guide](https://risingwavelabs.github.io/risingwave/).
-
-The [design docs](https://github.com/risingwavelabs/risingwave/blob/main/docs/README.md) covers some high-level ideas of how we built RisingWave.
+To learn how to develop RisingWave, and access high-level design docs, see the [RisingWave Developer Guide](https://risingwavelabs.github.io/risingwave/).
 
 ## Crate Docs
 
diff --git a/docs/rustdoc/rust.css b/docs/rustdoc/rust.css
index 71cf5e3df0004..9c76bb08c3898 100644
--- a/docs/rustdoc/rust.css
+++ b/docs/rustdoc/rust.css
@@ -1,18 +1,21 @@
 /* This file is copied from the Rust Project, which is dual-licensed under
-Apache 2.0 and MIT terms. */
+Apache 2.0 and MIT terms. https: //github.com/rust-lang/rust/blob/7d640b670e521a0491ea1e49082d1cb5632e2562/src/doc/rust.css
+*/
 
 /* General structure */
 
 body {
+	font-family: serif;
 	margin: 0 auto;
 	padding: 0 15px;
 	font-size: 18px;
-	color: #333;
+	color: #000;
 	line-height: 1.428571429;
 
 	-webkit-box-sizing: unset;
 	-moz-box-sizing: unset;
 	box-sizing: unset;
+	background: #fff;
 }
 @media (min-width: 768px) {
 	body {
@@ -20,6 +23,14 @@ body {
 	}
 }
 
+h1,
+h2,
+h3,
+h4,
+h5,
+h6 {
+	font-family: sans-serif;
+}
 h2, h3, h4, h5, h6 {
 	font-weight: 400;
 	line-height: 1.1;
@@ -37,8 +48,8 @@ h4, h5, h6 {
 	margin-bottom: 10px;
 	padding: 5px 10px;
 }
-h5, h6 {
-	color: black;
+h5,
+h6 {
 	text-decoration: underline;
 }
 
@@ -135,6 +146,31 @@ h1 a:link, h1 a:visited, h2 a:link, h2 a:visited,
 h3 a:link, h3 a:visited, h4 a:link, h4 a:visited,
 h5 a:link, h5 a:visited {color: black;}
 
+h1,
+h2,
+h3,
+h4,
+h5 {
+	/* This is needed to be able to position the doc-anchor. Ideally there
+	   would be a <div> around the whole document, but we don't have that. */
+	position: relative;
+}
+
+a.doc-anchor {
+	color: black;
+	display: none;
+	position: absolute;
+	left: -20px;
+	/* We add this padding so that when the cursor moves from the heading's text to the anchor,
+	   the anchor doesn't disappear. */
+	padding-right: 5px;
+	/* And this padding is used to make the anchor larger and easier to click on. */
+	padding-left: 3px;
+}
+
+*:hover>.doc-anchor {
+	display: block;
+}
 /* Code */
 
 pre, code {
diff --git a/e2e_test/backfill/sink/create_sink.slt b/e2e_test/backfill/sink/create_sink.slt
index 016e3bcb2049b..1eea929f3d50b 100644
--- a/e2e_test/backfill/sink/create_sink.slt
+++ b/e2e_test/backfill/sink/create_sink.slt
@@ -5,7 +5,7 @@ statement ok
 create table t (v1 int);
 
 statement ok
-SET STREAMING_RATE_LIMIT = 500;
+SET BACKFILL_RATE_LIMIT = 500;
 
 # Should finish in 20s
 statement ok
diff --git a/e2e_test/backfill/sink/different_pk_and_dist_key.slt b/e2e_test/backfill/sink/different_pk_and_dist_key.slt
index bc8256b28e62a..41c99f315b153 100644
--- a/e2e_test/backfill/sink/different_pk_and_dist_key.slt
+++ b/e2e_test/backfill/sink/different_pk_and_dist_key.slt
@@ -18,7 +18,7 @@ statement ok
 create materialized view m1 as select t.v1, t.v2, t.v3 from t join t2 using(v1);
 
 statement ok
-set streaming_rate_limit = 1;
+set backfill_rate_limit = 1;
 
 statement ok
 set background_ddl = true;
diff --git a/e2e_test/background_ddl/basic.slt b/e2e_test/background_ddl/basic.slt
index b0be505eb1dd9..3c98b6943610e 100644
--- a/e2e_test/background_ddl/basic.slt
+++ b/e2e_test/background_ddl/basic.slt
@@ -14,7 +14,7 @@ statement ok
 FLUSH;
 
 statement ok
-SET STREAMING_RATE_LIMIT=10000;
+SET BACKFILL_RATE_LIMIT=10000;
 
 statement ok
 CREATE MATERIALIZED VIEW m1 as SELECT * FROM t;
diff --git a/e2e_test/background_ddl/sim/basic.slt b/e2e_test/background_ddl/sim/basic.slt
index 35f5814fe8b4f..40cb606410130 100644
--- a/e2e_test/background_ddl/sim/basic.slt
+++ b/e2e_test/background_ddl/sim/basic.slt
@@ -14,7 +14,7 @@ statement ok
 FLUSH;
 
 statement ok
-SET STREAMING_RATE_LIMIT=4000;
+SET BACKFILL_RATE_LIMIT=4000;
 
 statement ok
 CREATE MATERIALIZED VIEW m1 as SELECT * FROM t;
diff --git a/e2e_test/batch/aggregate/scalar_as_agg.slt.part b/e2e_test/batch/aggregate/scalar_as_agg.slt.part
new file mode 100644
index 0000000000000..04e4f2809f26a
--- /dev/null
+++ b/e2e_test/batch/aggregate/scalar_as_agg.slt.part
@@ -0,0 +1,9 @@
+query T
+select aggregate:array_sum(v) from (values (3), (2), (1)) as t(v);
+----
+6
+
+query T
+select aggregate:array_sort(v) from (values (3), (2), (1)) as t(v);
+----
+{1,2,3}
diff --git a/e2e_test/batch/basic/union.slt.part b/e2e_test/batch/basic/union.slt.part
index 7dad5cf0ffe69..1db2f2ef4a502 100644
--- a/e2e_test/batch/basic/union.slt.part
+++ b/e2e_test/batch/basic/union.slt.part
@@ -2,33 +2,33 @@ statement ok
 SET RW_IMPLICIT_FLUSH TO true;
 
 statement ok
-create table t1 (v1 int, v2 bigint);
+create table t1 (v1 int, v2 bigint, v4 int);
 
 statement ok
-create table t2 (v1 int, v3 int);
+create table t2 (v1 int, v3 int, v4 int);
 
 statement ok
-insert into t1 values(1, 2);
+insert into t1 values(1, 2, 3);
 
 statement ok
-insert into t2 values(1, 2);
+insert into t2 values(1, 2, 3);
 
-query II
+query III
 select * from t1 union select * from t2
 ----
-1 2
+1 2 3
 
-query II
+query III
 select * from t1 union all select * from t2
 ----
-1 2
-1 2
+1 2 3
+1 2 3
 
-query II
+query III
 select * from t1 union all select * from t2 order by v1
 ----
-1 2
-1 2
+1 2 3
+1 2 3
 
 statement error
 select * from t1 union all select * from t2 order by v1 + 1
@@ -69,9 +69,55 @@ NULL
 statement error
 select null union all select null select union 1
 
+query II
+select * from t1 union all corresponding select * from t2 order by v1
+----
+1 3
+1 3
+
+query II
+select * from t1 union corresponding select v4, v3 as v1 from t2 order by v1
+----
+1 3
+2 3
+
+query II
+select * from t1 union all corresponding by (v4, v1) select * from t2
+----
+3 1
+3 1
+
+query II
+select * from t1 union corresponding by (v4) select * from t2
+----
+3
+
+statement error
+select * from t1 union corresponding by (vxx) select * from t2
+----
+db error: ERROR: Failed to run the query
+
+Caused by:
+  Invalid input syntax: Column name `vxx` in CORRESPONDING BY is not found in a side of the UNION operation. It shall be included in both sides.
+
+
+statement ok
+create table txx (vxx int);
+
+statement error
+select * from t1 union corresponding select * from txx
+----
+db error: ERROR: Failed to run the query
+
+Caused by:
+  Invalid input syntax: When CORRESPONDING is specified, at least one column of the left side shall have a column name that is the column name of some column of the right side in a UNION operation. Left side query column list: ("v1", "v2", "v4"). Right side query column list: ("vxx").
+
 
 statement ok
 drop table t1;
 
 statement ok
 drop table t2;
+
+statement ok
+drop table txx;
diff --git a/e2e_test/batch/catalog/pg_settings.slt.part b/e2e_test/batch/catalog/pg_settings.slt.part
index f405cc71c2c0d..081a30cd90cb1 100644
--- a/e2e_test/batch/catalog/pg_settings.slt.part
+++ b/e2e_test/batch/catalog/pg_settings.slt.part
@@ -16,7 +16,9 @@ postmaster enable_tracing
 postmaster license_key
 postmaster max_concurrent_creating_streaming_jobs
 postmaster pause_on_next_bootstrap
+postmaster time_travel_retention_ms
 user application_name
+user backfill_rate_limit
 user background_ddl
 user batch_enable_distributed_dml
 user batch_parallelism
@@ -52,10 +54,10 @@ user server_encoding
 user server_version
 user server_version_num
 user sink_decouple
+user source_rate_limit
 user standard_conforming_strings
 user statement_timeout
 user streaming_parallelism
-user streaming_rate_limit
 user streaming_use_arrangement_backfill
 user synchronize_seqscans
 user timezone
diff --git a/e2e_test/batch/functions/trigonometric_funcs.slt.part b/e2e_test/batch/functions/trigonometric_funcs.slt.part
index ae30880a155ba..cb3e26115670d 100644
--- a/e2e_test/batch/functions/trigonometric_funcs.slt.part
+++ b/e2e_test/batch/functions/trigonometric_funcs.slt.part
@@ -659,3 +659,48 @@ query R
 SELECT abs(abs(asind(-0.5) + 30)) < 1e-14
 ----
 t
+
+query R
+SELECT abs(acosd(-1) - 180.0) < 1e-14
+----
+t
+
+query R
+SELECT abs(acosd(-0.75) - 138.59037789072914) < 1e-14
+----
+t
+
+query R
+SELECT abs(acosd(-0.5) - 120.0) < 1e-14
+----
+t
+
+query R
+SELECT abs(acosd(-0.25) - 104.47751218592992) < 1e-14
+----
+t
+
+query R
+SELECT abs(acosd(0) - 90.0) < 1e-14
+----
+t
+
+query R
+SELECT abs(acosd(0.25) - 75.52248781407008) < 1e-14
+----
+t
+
+query R
+SELECT abs(acosd(0.5) - 59.99999999999999) < 1e-14
+----
+t
+
+query R
+SELECT abs(acosd(0.75) - 41.40962210927086) < 1e-14
+----
+t
+
+query R
+SELECT abs(acosd(1) - 0.0) < 1e-14
+----
+t
diff --git a/e2e_test/commands/sr_register b/e2e_test/commands/sr_register
new file mode 100755
index 0000000000000..57dc65e50610d
--- /dev/null
+++ b/e2e_test/commands/sr_register
@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# Register a schema to schema registry
+#
+# Usage: sr_register <subject> <schema>
+#
+# https://docs.confluent.io/platform/current/schema-registry/develop/api.html#post--subjects-(string-%20subject)-versions
+
+# Validate arguments
+if [[ $# -ne 2 ]]; then
+    echo "Usage: sr_register <subject> <schema>"
+    exit 1
+fi
+
+subject="$1"
+schema="$2"
+
+
+if [[ -z $subject || -z $schema ]]; then
+    echo "Error: Arguments cannot be empty"
+    exit 1
+fi
+
+echo "$schema" | jq '{"schema": tojson}' \
+| curl -X POST -H 'content-type:application/vnd.schemaregistry.v1+json' -d @- "${RISEDEV_SCHEMA_REGISTRY_URL}/subjects/${subject}/versions"
diff --git a/e2e_test/ddl/drop/drop_creating_mv.slt b/e2e_test/ddl/drop/drop_creating_mv.slt
new file mode 100644
index 0000000000000..e9d8423f3cdec
--- /dev/null
+++ b/e2e_test/ddl/drop/drop_creating_mv.slt
@@ -0,0 +1,76 @@
+statement ok
+create table t(v1 int);
+
+statement ok
+insert into t select * from generate_series(1, 10000);
+
+statement ok
+flush;
+
+statement ok
+set backfill_rate_limit=1;
+
+############## Test drop foreground mv
+onlyif can-use-recover
+system ok
+risedev psql -c 'create materialized view m1 as select * from t;' &
+
+onlyif can-use-recover
+sleep 5s
+
+onlyif can-use-recover
+statement ok
+drop materialized view m1;
+
+############## Test drop background mv BEFORE recovery
+statement ok
+set background_ddl=true;
+
+onlyif can-use-recover
+statement ok
+create materialized view m1 as select * from t;
+
+onlyif can-use-recover
+sleep 5s
+
+onlyif can-use-recover
+statement ok
+drop materialized view m1;
+
+############## Test drop background mv AFTER recovery
+statement ok
+set background_ddl=true;
+
+onlyif can-use-recover
+statement ok
+create materialized view m1 as select * from t;
+
+onlyif can-use-recover
+sleep 5s
+
+onlyif can-use-recover
+statement ok
+recover;
+
+onlyif can-use-recover
+sleep 10s
+
+onlyif can-use-recover
+statement ok
+drop materialized view m1;
+
+############## Make sure the mv can still be successfully created later.
+statement ok
+set backfill_rate_limit=default;
+
+statement ok
+set background_ddl=false;
+
+statement ok
+create materialized view m1 as select * from t;
+
+statement ok
+drop materialized view m1;
+
+statement ok
+drop table t;
\ No newline at end of file
diff --git a/e2e_test/ddl/secret.slt b/e2e_test/ddl/secret.slt
index 7c11e2e6245a3..62a5a09add668 100644
--- a/e2e_test/ddl/secret.slt
+++ b/e2e_test/ddl/secret.slt
@@ -1,3 +1,33 @@
+statement ok
+ALTER SYSTEM SET license_key TO '';
+
+statement error
+create secret secret_1 with (
+  backend = 'fake-backend'
+) as 'demo_secret';
+----
+db error: ERROR: Failed to run the query
+
+Caused by:
+  feature SecretManagement is only available for tier Paid and above, while the current tier is Free
+
+Hint: You may want to set a license key with `ALTER SYSTEM SET license_key = '...';` command.
+
+
+statement error
+drop secret secret_1;
+----
+db error: ERROR: Failed to run the query
+
+Caused by:
+  feature SecretManagement is only available for tier Paid and above, while the current tier is Free
+
+Hint: You may want to set a license key with `ALTER SYSTEM SET license_key = '...';` command.
+
+
+statement ok
+ALTER SYSTEM SET license_key TO DEFAULT;
+
 statement error secret backend "fake-backend" is not supported
 create secret secret_1 with (
   backend = 'fake-backend'
diff --git a/e2e_test/ddl/table/generated_columns.slt.part b/e2e_test/ddl/table/generated_columns.slt.part
index 2271522a47fd9..e1be2f26417ee 100644
--- a/e2e_test/ddl/table/generated_columns.slt.part
+++ b/e2e_test/ddl/table/generated_columns.slt.part
@@ -58,6 +58,21 @@ select * from t2;
 1 2
 2 3
 
+statement error
+alter table t2 drop column v1;
+----
+db error: ERROR: Failed to run the query
+
+Caused by:
+  failed to drop column "v1" because it's referenced by a generated column "v2"
+
+
+statement ok
+alter table t2 drop column v2;
+
+statement ok
+alter table t2 drop column v1;
+
 statement ok
 drop table t2;
 
diff --git a/e2e_test/ddl/throttle.slt b/e2e_test/ddl/throttle.slt
index 9b6c2f053bf63..e8a33b0bfb433 100644
--- a/e2e_test/ddl/throttle.slt
+++ b/e2e_test/ddl/throttle.slt
@@ -1,20 +1,22 @@
-# streaming_rate_limit also applies to create sink and create source, please refer to
-# e2e_test/source/basic/kafka.slt and e2e_test/sink/kafka/create_sink.slt for this part
+# streaming_rate_limit applies to create source, please refer to
+# e2e_test/source/basic/kafka.slt.
+# backfill_rate_limit applies to create sink, please refer to
+# e2e_test/sink/kafka/create_sink.slt.
 
 statement ok
 create table t1 (v1 int);
 
 # tracked in https://github.com/risingwavelabs/risingwave/issues/13474
-# create with duplicate streaming_rate_limit
+# create with duplicate backfill_rate_limit
 statement error Duplicated option
-create materialized view mv1 with (streaming_rate_limit = 1000, streaming_rate_limit = 2000) as select * from t1;
+create materialized view mv1 with (backfill_rate_limit = 1000, backfill_rate_limit = 2000) as select * from t1;
 
 # create with unknown fields
 statement error unexpected options in WITH clause
-create materialized view mv1 with (streaming_rate_limit = 1000, unknown_field = 2000) as select * from t1;
+create materialized view mv1 with (backfill_rate_limit = 1000, unknown_field = 2000) as select * from t1;
 
 statement ok
-create materialized view mv1 with (streaming_rate_limit = 1000) as select * from t1;
+create materialized view mv1 with (backfill_rate_limit = 1000) as select * from t1;
 
 statement ok
 drop materialized view mv1;
diff --git a/e2e_test/error_ui/simple/recovery.slt b/e2e_test/error_ui/simple/recovery.slt
index e3830be6d25c8..526e9d2f10227 100644
--- a/e2e_test/error_ui/simple/recovery.slt
+++ b/e2e_test/error_ui/simple/recovery.slt
@@ -25,7 +25,7 @@ with error as (
     limit 1
 )
 select
-case when error like '%Actor % exited unexpectedly: Executor error: %Numeric out of range%' then 'ok'
+case when error like '%get error from control stream, in worker node %: %Actor % exited unexpectedly: Executor error: %Numeric out of range%' then 'ok'
      else error
 end as result
 from error;
diff --git a/e2e_test/iceberg/test_case/append_only_with_checkpoint_interval.slt b/e2e_test/iceberg/test_case/append_only_with_checkpoint_interval.slt
index 7d1f9254e4683..0a8a042a86ac4 100644
--- a/e2e_test/iceberg/test_case/append_only_with_checkpoint_interval.slt
+++ b/e2e_test/iceberg/test_case/append_only_with_checkpoint_interval.slt
@@ -29,7 +29,7 @@ CREATE SINK sink1 AS select * from mv1 WITH (
     table.name = 't1',
     catalog.name = 'demo',
     catalog.type = 'storage',
-    warehouse.path = 's3://icebergdata/demo',
+    warehouse.path = 's3a://icebergdata/demo',
     s3.endpoint = 'http://127.0.0.1:9301',
     s3.region = 'us-east-1',
     s3.access.key = 'hummockadmin',
@@ -45,7 +45,7 @@ CREATE SINK sink2 AS select * from mv1 WITH (
     table.name = 't2',
     catalog.name = 'demo',
     catalog.type = 'storage',
-    warehouse.path = 's3://icebergdata/demo',
+    warehouse.path = 's3a://icebergdata/demo',
     s3.endpoint = 'http://127.0.0.1:9301',
     s3.region = 'us-east-1',
     s3.access.key = 'hummockadmin',
diff --git a/e2e_test/iceberg/test_case/cdc/load.slt b/e2e_test/iceberg/test_case/cdc/load.slt
index e9f1d815d20cb..df0c319990374 100644
--- a/e2e_test/iceberg/test_case/cdc/load.slt
+++ b/e2e_test/iceberg/test_case/cdc/load.slt
@@ -28,7 +28,7 @@ CREATE SINK s1 AS select * from products WITH (
     database.name = 'demo_db',
     table.name = 'demo_table',
     catalog.type = 'storage',
-    warehouse.path = 's3://icebergdata/demo',
+    warehouse.path = 's3a://icebergdata/demo',
     s3.endpoint = 'http://127.0.0.1:9301',
     s3.region = 'us-east-1',
     s3.access.key = 'hummockadmin',
diff --git a/e2e_test/iceberg/test_case/iceberg_sink_no_partition_append_only_table.slt b/e2e_test/iceberg/test_case/iceberg_sink_no_partition_append_only_table.slt
index 505bf0ba7d21c..658a9ae8563e1 100644
--- a/e2e_test/iceberg/test_case/iceberg_sink_no_partition_append_only_table.slt
+++ b/e2e_test/iceberg/test_case/iceberg_sink_no_partition_append_only_table.slt
@@ -28,7 +28,7 @@ CREATE SINK s6 AS select * from mv6 WITH (
     table.name = 'no_partition_append_only_table',
     catalog.name = 'demo',
     catalog.type = 'storage',
-    warehouse.path = 's3://icebergdata/demo',
+    warehouse.path = 's3a://icebergdata/demo',
     s3.endpoint = 'http://127.0.0.1:9301',
     s3.region = 'us-east-1',
     s3.access.key = 'hummockadmin',
diff --git a/e2e_test/iceberg/test_case/iceberg_sink_no_partition_append_only_table_verify.slt b/e2e_test/iceberg/test_case/iceberg_sink_no_partition_append_only_table_verify.slt
index 74629053344b5..a789782367261 100644
--- a/e2e_test/iceberg/test_case/iceberg_sink_no_partition_append_only_table_verify.slt
+++ b/e2e_test/iceberg/test_case/iceberg_sink_no_partition_append_only_table_verify.slt
@@ -7,7 +7,7 @@ WITH (
     s3.access.key = 'hummockadmin',
     s3.secret.key = 'hummockadmin',
     catalog.type = 'storage',
-    warehouse.path = 's3://icebergdata/demo',
+    warehouse.path = 's3a://icebergdata/demo',
     database.name = 'demo_db',
     table.name = 'no_partition_append_only_table',
 );
diff --git a/e2e_test/iceberg/test_case/iceberg_sink_no_partition_upsert_table.slt b/e2e_test/iceberg/test_case/iceberg_sink_no_partition_upsert_table.slt
index b59c7013290ec..1ff15295c745a 100644
--- a/e2e_test/iceberg/test_case/iceberg_sink_no_partition_upsert_table.slt
+++ b/e2e_test/iceberg/test_case/iceberg_sink_no_partition_upsert_table.slt
@@ -16,7 +16,7 @@ CREATE SINK s6 AS select mv6.id as id, mv6.v1 as v1, mv6.v2 as v2, mv6.v3 as v3,
     database.name = 'demo_db',
     table.name = 'no_partition_upsert_table',
     catalog.type = 'storage',
-    warehouse.path = 's3://icebergdata/demo',
+    warehouse.path = 's3a://icebergdata/demo',
     s3.endpoint = 'http://127.0.0.1:9301',
     s3.region = 'us-east-1',
     s3.access.key = 'hummockadmin',
diff --git a/e2e_test/iceberg/test_case/iceberg_sink_partition_append_only_table.slt b/e2e_test/iceberg/test_case/iceberg_sink_partition_append_only_table.slt
index da55b35626ff3..bd328f7834d6f 100644
--- a/e2e_test/iceberg/test_case/iceberg_sink_partition_append_only_table.slt
+++ b/e2e_test/iceberg/test_case/iceberg_sink_partition_append_only_table.slt
@@ -28,7 +28,7 @@ CREATE SINK s6 AS select * from mv6 WITH (
     table.name = 'partition_append_only_table',
     catalog.name = 'demo',
     catalog.type = 'storage',
-    warehouse.path = 's3://icebergdata/demo',
+    warehouse.path = 's3a://icebergdata/demo',
     s3.endpoint = 'http://127.0.0.1:9301',
     s3.region = 'us-east-1',
     s3.access.key = 'hummockadmin',
diff --git a/e2e_test/iceberg/test_case/iceberg_sink_partition_append_only_table_verify.slt b/e2e_test/iceberg/test_case/iceberg_sink_partition_append_only_table_verify.slt
index 4e6beb709f920..58d848b54cb21 100644
--- a/e2e_test/iceberg/test_case/iceberg_sink_partition_append_only_table_verify.slt
+++ b/e2e_test/iceberg/test_case/iceberg_sink_partition_append_only_table_verify.slt
@@ -7,7 +7,7 @@ WITH (
     s3.access.key = 'hummockadmin',
     s3.secret.key = 'hummockadmin',
     catalog.type = 'storage',
-    warehouse.path = 's3://icebergdata/demo',
+    warehouse.path = 's3a://icebergdata/demo',
     database.name = 'demo_db',
     table.name = 'partition_append_only_table',
 );
diff --git a/e2e_test/iceberg/test_case/iceberg_sink_partition_upsert_table.slt b/e2e_test/iceberg/test_case/iceberg_sink_partition_upsert_table.slt
index 20800c1b4787d..1435e03877f40 100644
--- a/e2e_test/iceberg/test_case/iceberg_sink_partition_upsert_table.slt
+++ b/e2e_test/iceberg/test_case/iceberg_sink_partition_upsert_table.slt
@@ -16,7 +16,7 @@ CREATE SINK s6 AS select mv6.id as id, mv6.v1 as v1, mv6.v2 as v2, mv6.v3 as v3,
     database.name = 'demo_db',
     table.name = 'partition_upsert_table',
     catalog.type = 'storage',
-    warehouse.path = 's3://icebergdata/demo',
+    warehouse.path = 's3a://icebergdata/demo',
     s3.endpoint = 'http://127.0.0.1:9301',
     s3.region = 'us-east-1',
     s3.access.key = 'hummockadmin',
diff --git a/e2e_test/iceberg/test_case/iceberg_sink_range_partition_append_only_table.slt b/e2e_test/iceberg/test_case/iceberg_sink_range_partition_append_only_table.slt
index 2a8d562b66068..e71fe49f31f9c 100644
--- a/e2e_test/iceberg/test_case/iceberg_sink_range_partition_append_only_table.slt
+++ b/e2e_test/iceberg/test_case/iceberg_sink_range_partition_append_only_table.slt
@@ -28,7 +28,7 @@ CREATE SINK s6 AS select * from mv6 WITH (
     table.name = 'range_partition_append_only_table',
     catalog.name = 'demo',
     catalog.type = 'storage',
-    warehouse.path = 's3://icebergdata/demo',
+    warehouse.path = 's3a://icebergdata/demo',
     s3.endpoint = 'http://127.0.0.1:9301',
     s3.region = 'us-east-1',
     s3.access.key = 'hummockadmin',
diff --git a/e2e_test/iceberg/test_case/iceberg_sink_range_partition_append_only_table_verify.slt b/e2e_test/iceberg/test_case/iceberg_sink_range_partition_append_only_table_verify.slt
index 9d03d99aada1b..eaa6b3d95eb6a 100644
--- a/e2e_test/iceberg/test_case/iceberg_sink_range_partition_append_only_table_verify.slt
+++ b/e2e_test/iceberg/test_case/iceberg_sink_range_partition_append_only_table_verify.slt
@@ -7,7 +7,7 @@ WITH (
     s3.access.key = 'hummockadmin',
     s3.secret.key = 'hummockadmin',
     catalog.type = 'storage',
-    warehouse.path = 's3://icebergdata/demo',
+    warehouse.path = 's3a://icebergdata/demo',
     database.name = 'demo_db',
     table.name = 'range_partition_append_only_table',
 );
diff --git a/e2e_test/iceberg/test_case/iceberg_sink_range_partition_upsert_table.slt b/e2e_test/iceberg/test_case/iceberg_sink_range_partition_upsert_table.slt
index 6e963b0e16615..daf1ceaa40bf8 100644
--- a/e2e_test/iceberg/test_case/iceberg_sink_range_partition_upsert_table.slt
+++ b/e2e_test/iceberg/test_case/iceberg_sink_range_partition_upsert_table.slt
@@ -16,7 +16,7 @@ CREATE SINK s6 AS select mv6.id as id, mv6.v1 as v1, mv6.v2 as v2, mv6.v3 as v3,
     database.name = 'demo_db',
     table.name = 'range_partition_upsert_table',
     catalog.type = 'storage',
-    warehouse.path = 's3://icebergdata/demo',
+    warehouse.path = 's3a://icebergdata/demo',
     s3.endpoint = 'http://127.0.0.1:9301',
     s3.region = 'us-east-1',
     s3.access.key = 'hummockadmin',
diff --git a/e2e_test/s3/fs_parquet_source.py b/e2e_test/s3/fs_parquet_source.py
new file mode 100644
index 0000000000000..64060928c7755
--- /dev/null
+++ b/e2e_test/s3/fs_parquet_source.py
@@ -0,0 +1,137 @@
+import os
+import sys
+import random
+import psycopg2
+import json
+import pyarrow as pa
+import pyarrow.parquet as pq
+import pandas as pd
+from datetime import datetime, timezone
+from time import sleep
+from minio import Minio
+from random import uniform
+
+def gen_data(file_num, item_num_per_file):
+    assert item_num_per_file % 2 == 0, \
+        f'item_num_per_file should be even to ensure sum(mark) == 0: {item_num_per_file}'
+    return [
+        [{
+            'id': file_id * item_num_per_file + item_id,
+            'name': f'{file_id}_{item_id}_{file_id * item_num_per_file + item_id}',
+            'sex': item_id % 2,
+            'mark': (-1) ** (item_id % 2),
+            'test_int': pa.scalar(1, type=pa.int32()),
+            'test_real': pa.scalar(4.0, type=pa.float32()),
+            'test_double_precision': pa.scalar(5.0, type=pa.float64()),
+            'test_varchar': pa.scalar('7', type=pa.string()),
+            'test_bytea': pa.scalar(b'\xDe00BeEf', type=pa.binary()),
+            'test_date': pa.scalar(datetime.now().date(), type=pa.date32()),
+            'test_time': pa.scalar(datetime.now().time(), type=pa.time64('us')),
+            'test_timestamp': pa.scalar(datetime.now().timestamp() * 1000000, type=pa.timestamp('us')),
+            'test_timestamptz': pa.scalar(datetime.now().timestamp() * 1000, type=pa.timestamp('us', tz='+00:00')),
+        } for item_id in range(item_num_per_file)]
+        for file_id in range(file_num)
+    ]
+
+def do_test(config, file_num, item_num_per_file, prefix):
+    conn = psycopg2.connect(
+        host="localhost",
+        port="4566",
+        user="root",
+        database="dev"
+    )
+
+    # Open a cursor to execute SQL statements
+    cur = conn.cursor()
+
+    def _table():
+        return 's3_test_parquet'
+
+    # Execute a SELECT statement
+    cur.execute(f'''CREATE TABLE {_table()}(
+        id bigint primary key,
+        name TEXT,
+        sex bigint,
+        mark bigint,
+        test_int int,
+        test_real real,
+        test_double_precision double precision,
+        test_varchar varchar,
+        test_bytea bytea,
+        test_date date,
+        test_time time,
+        test_timestamp timestamp,
+        test_timestamptz timestamptz,
+    ) WITH (
+        connector = 's3_v2',
+        match_pattern = '*.parquet',
+        s3.region_name = '{config['S3_REGION']}',
+        s3.bucket_name = '{config['S3_BUCKET']}',
+        s3.credentials.access = '{config['S3_ACCESS_KEY']}',
+        s3.credentials.secret = '{config['S3_SECRET_KEY']}',
+        s3.endpoint_url = 'https://{config['S3_ENDPOINT']}'
+    ) FORMAT PLAIN ENCODE PARQUET;''')
+
+    total_rows = file_num * item_num_per_file
+    MAX_RETRIES = 40
+    for retry_no in range(MAX_RETRIES):
+        cur.execute(f'select count(*) from {_table()}')
+        result = cur.fetchone()
+        if result[0] == total_rows:
+            break
+        print(f"[retry {retry_no}] Now got {result[0]} rows in table, {total_rows} expected, wait 10s")
+        sleep(10)
+
+    stmt = f'select count(*), sum(id) from {_table()}'
+    print(f'Execute {stmt}')
+    cur.execute(stmt)
+    result = cur.fetchone()
+
+    print('Got:', result)
+
+    def _assert_eq(field, got, expect):
+        assert got == expect, f'{field} assertion failed: got {got}, expect {expect}.'
+
+    _assert_eq('count(*)', result[0], total_rows)
+    _assert_eq('sum(id)', result[1], (total_rows - 1) * total_rows / 2)
+
+    print('Test pass')
+
+    cur.execute(f'drop table {_table()}')
+    cur.close()
+    conn.close()
+
+
+if __name__ == "__main__":
+    FILE_NUM = 10
+    ITEM_NUM_PER_FILE = 2000
+    data = gen_data(FILE_NUM, ITEM_NUM_PER_FILE)
+
+    config = json.loads(os.environ["S3_SOURCE_TEST_CONF"])
+    client = Minio(
+        config["S3_ENDPOINT"],
+        access_key=config["S3_ACCESS_KEY"],
+        secret_key=config["S3_SECRET_KEY"],
+        secure=True,
+    )
+    run_id = str(random.randint(1000, 9999))
+    _local = lambda idx: f'data_{idx}.parquet'
+    _s3 = lambda idx: f"{run_id}_data_{idx}.parquet"
+
+    # put s3 files
+    for idx, file_data in enumerate(data):
+        table = pa.Table.from_pandas(pd.DataFrame(file_data))
+        pq.write_table(table, _local(idx))
+
+        client.fput_object(
+            config["S3_BUCKET"],
+            _s3(idx),
+            _local(idx)
+        )
+
+    # do test
+    do_test(config, FILE_NUM, ITEM_NUM_PER_FILE, run_id)
+
+    # clean up s3 files
+    for idx, _ in enumerate(data):
+       client.remove_object(config["S3_BUCKET"], _s3(idx))
\ No newline at end of file
diff --git a/e2e_test/s3/fs_source_v2.py b/e2e_test/s3/fs_source_v2.py
index 760b8d07a09a5..6706d4b6d4a92 100644
--- a/e2e_test/s3/fs_source_v2.py
+++ b/e2e_test/s3/fs_source_v2.py
@@ -43,7 +43,7 @@ def format_csv(data, with_header):
         csv_files.append(ostream.getvalue())
     return csv_files
 
-def do_test(config, file_num, item_num_per_file, prefix, fmt):
+def do_test(config, file_num, item_num_per_file, prefix, fmt, need_drop_table=True):
     conn = psycopg2.connect(
         host="localhost",
         port="4566",
@@ -106,10 +106,16 @@ def _assert_eq(field, got, expect):
 
     print('Test pass')
 
-    cur.execute(f'drop table {_table()}')
+    if need_drop_table:
+        cur.execute(f'drop table {_table()}')
     cur.close()
     conn.close()
 
+FORMATTER = {
+        'json': format_json,
+        'csv_with_header': partial(format_csv, with_header=True),
+        'csv_without_header': partial(format_csv, with_header=False),
+    }
 
 if __name__ == "__main__":
     FILE_NUM = 4001
@@ -117,11 +123,6 @@ def _assert_eq(field, got, expect):
     data = gen_data(FILE_NUM, ITEM_NUM_PER_FILE)
 
     fmt = sys.argv[1]
-    FORMATTER = {
-        'json': format_json,
-        'csv_with_header': partial(format_csv, with_header=True),
-        'csv_without_header': partial(format_csv, with_header=False),
-    }
     assert fmt in FORMATTER, f"Unsupported format: {fmt}"
     formatted_files = FORMATTER[fmt](data)
 
diff --git a/e2e_test/s3/fs_source_v2_new_file.py b/e2e_test/s3/fs_source_v2_new_file.py
new file mode 100644
index 0000000000000..a7ae53f3a37dd
--- /dev/null
+++ b/e2e_test/s3/fs_source_v2_new_file.py
@@ -0,0 +1,86 @@
+from fs_source_v2 import gen_data, FORMATTER, do_test
+import json
+import os
+import random
+import psycopg2
+import time
+from minio import Minio
+
+
+def upload_to_s3_bucket(config, minio_client, run_id, files, start_bias):
+    _local = lambda idx, start_bias: f"data_{idx + start_bias}.{fmt}"
+    _s3 = lambda idx, start_bias: f"{run_id}_data_{idx + start_bias}.{fmt}"
+    for idx, file_str in enumerate(files):
+        with open(_local(idx, start_bias), "w") as f:
+            f.write(file_str)
+            os.fsync(f.fileno())
+
+        minio_client.fput_object(
+            config["S3_BUCKET"], _s3(idx, start_bias), _local(idx, start_bias)
+        )
+
+
+def check_for_new_files(file_num, item_num_per_file, fmt):
+    conn = psycopg2.connect(host="localhost", port="4566", user="root", database="dev")
+
+    # Open a cursor to execute SQL statements
+    cur = conn.cursor()
+
+    def _table():
+        return f"s3_test_{fmt}"
+
+    total_rows = file_num * item_num_per_file
+
+    MAX_RETRIES = 40
+    for retry_no in range(MAX_RETRIES):
+        cur.execute(f"select count(*) from {_table()}")
+        result = cur.fetchone()
+        if result[0] == total_rows:
+            return True
+        print(
+            f"[retry {retry_no}] Now got {result[0]} rows in table, {total_rows} expected, wait 10s"
+        )
+        time.sleep(10)
+    return False
+
+
+if __name__ == "__main__":
+    FILE_NUM = 101
+    ITEM_NUM_PER_FILE = 2
+    data = gen_data(FILE_NUM, ITEM_NUM_PER_FILE)
+    fmt = "json"
+
+    split_idx = 51
+    data_batch1 = data[:split_idx]
+    data_batch2 = data[split_idx:]
+
+    config = json.loads(os.environ["S3_SOURCE_TEST_CONF"])
+    client = Minio(
+        config["S3_ENDPOINT"],
+        access_key=config["S3_ACCESS_KEY"],
+        secret_key=config["S3_SECRET_KEY"],
+        secure=True,
+    )
+    run_id = str(random.randint(1000, 9999))
+    print(f"S3 Source New File Test: run ID: {run_id} to bucket {config['S3_BUCKET']}")
+
+    formatted_batch1 = FORMATTER[fmt](data_batch1)
+    upload_to_s3_bucket(config, client, run_id, formatted_batch1, 0)
+
+    do_test(
+        config, len(data_batch1), ITEM_NUM_PER_FILE, run_id, fmt, need_drop_table=False
+    )
+
+    formatted_batch2 = FORMATTER[fmt](data_batch2)
+    upload_to_s3_bucket(config, client, run_id, formatted_batch2, split_idx)
+
+    success_flag = check_for_new_files(FILE_NUM, ITEM_NUM_PER_FILE, fmt)
+    if success_flag:
+        print("Test(add new file) pass")
+    else:
+        print("Test(add new file) fail")
+
+    _s3 = lambda idx, start_bias: f"{run_id}_data_{idx + start_bias}.{fmt}"
+    # clean up s3 files
+    for idx, _ in enumerate(data):
+        client.remove_object(config["S3_BUCKET"], _s3(idx, 0))
diff --git a/e2e_test/sink/deltalake_rust_sink.slt b/e2e_test/sink/deltalake_rust_sink.slt
index a658520cdeb44..74dca623a9d0a 100644
--- a/e2e_test/sink/deltalake_rust_sink.slt
+++ b/e2e_test/sink/deltalake_rust_sink.slt
@@ -4,6 +4,11 @@ CREATE TABLE t6 (v1 int primary key, v2 smallint, v3 bigint, v4 real, v5 float,
 statement ok
 CREATE MATERIALIZED VIEW mv6 AS SELECT * FROM t6;
 
+statement ok
+CREATE SECRET deltalake_s3_secret_key WITH (
+  backend = 'meta'
+) as 'hummockadmin';
+
 statement ok
 create sink s6 as select * from mv6
 with (
@@ -12,7 +17,7 @@ with (
     force_append_only = 'true',
     location = 's3a://deltalake/deltalake-test',
     s3.access.key = 'hummockadmin',
-    s3.secret.key = 'hummockadmin',
+    s3.secret.key = secret deltalake_s3_secret_key,
     s3.endpoint = 'http://127.0.0.1:9301'
 );
 
@@ -25,6 +30,9 @@ FLUSH;
 statement ok
 DROP SINK s6;
 
+statement ok
+DROP SECRET deltalake_s3_secret_key;
+
 statement ok
 DROP MATERIALIZED VIEW mv6;
 
diff --git a/e2e_test/sink/iceberg_sink.slt b/e2e_test/sink/iceberg_sink.slt
index 05cf2d1108b36..08fa23dd839c6 100644
--- a/e2e_test/sink/iceberg_sink.slt
+++ b/e2e_test/sink/iceberg_sink.slt
@@ -4,15 +4,25 @@ CREATE TABLE t6 (v1 int primary key, v2 bigint, v3 varchar);
 statement ok
 CREATE MATERIALIZED VIEW mv6 AS SELECT * FROM t6;
 
+statement ok
+CREATE SECRET iceberg_s3_access_key WITH (
+  backend = 'meta'
+) as 'hummockadmin';
+
+statement ok
+CREATE SECRET iceberg_s3_secret_key WITH (
+  backend = 'meta'
+) as 'hummockadmin';
+
 statement ok
 CREATE SINK s6 AS select mv6.v1 as v1, mv6.v2 as v2, mv6.v3 as v3 from mv6 WITH (
     connector = 'iceberg',
     type = 'upsert',
     primary_key = 'v1',
-    warehouse.path = 's3://iceberg',
+    warehouse.path = 's3a://iceberg',
     s3.endpoint = 'http://127.0.0.1:9301',
-    s3.access.key = 'hummockadmin',
-    s3.secret.key = 'hummockadmin',
+    s3.access.key = secret iceberg_s3_access_key,
+    s3.secret.key = secret iceberg_s3_secret_key,
     s3.region = 'us-east-1',
     catalog.name = 'demo',
     catalog.type = 'storage',
@@ -23,10 +33,10 @@ CREATE SINK s6 AS select mv6.v1 as v1, mv6.v2 as v2, mv6.v3 as v3 from mv6 WITH
 statement ok
 CREATE SOURCE iceberg_demo_source WITH (
     connector = 'iceberg',
-    warehouse.path = 's3://iceberg',
+    warehouse.path = 's3a://iceberg',
     s3.endpoint = 'http://127.0.0.1:9301',
-    s3.access.key = 'hummockadmin',
-    s3.secret.key = 'hummockadmin',
+    s3.access.key = secret iceberg_s3_access_key,
+    s3.secret.key = secret iceberg_s3_secret_key,
     s3.region = 'us-east-1',
     catalog.name = 'demo',
     catalog.type = 'storage',
diff --git a/e2e_test/sink/kafka/create_sink.slt b/e2e_test/sink/kafka/create_sink.slt
index be4c39c7cb1c6..338465c471af9 100644
--- a/e2e_test/sink/kafka/create_sink.slt
+++ b/e2e_test/sink/kafka/create_sink.slt
@@ -153,7 +153,7 @@ create sink multiple_pk_throttle from t_kafka with (
     topic = 'test-rw-sink-debezium',
     type = 'debezium',
     primary_key = 'id,v_varchar',
-    streaming_rate_limit = 200
+    backfill_rate_limit = 200
 );
 
 statement ok
@@ -165,7 +165,7 @@ create sink multiple_pk_throttle_1
     topic = 'test-rw-sink-debezium',
     type = 'debezium',
     primary_key = 'id,v_varchar',
-    streaming_rate_limit = 200
+    backfill_rate_limit = 200
 );
 
 statement ok
diff --git a/e2e_test/sink/sink_into_table/alter_column.slt b/e2e_test/sink/sink_into_table/alter_column.slt
new file mode 100644
index 0000000000000..ce003ebef3fa2
--- /dev/null
+++ b/e2e_test/sink/sink_into_table/alter_column.slt
@@ -0,0 +1,201 @@
+statement ok
+SET RW_IMPLICIT_FLUSH TO true;
+
+statement ok
+create table t_simple_1 (v1 int);
+
+statement ok
+create table m_simple (v1 int primary key);
+
+statement ok
+create sink s_simple_1 into m_simple as select v1 from t_simple_1;
+
+statement ok
+insert into t_simple_1 values (1), (2), (3);
+
+statement ok
+flush;
+
+query I rowsort
+select * from m_simple;
+----
+1
+2
+3
+
+statement ok
+alter table m_simple add column v2 int;
+
+statement ok
+insert into t_simple_1 values (4);
+
+statement ok
+flush;
+
+query II rowsort
+select * from m_simple;
+----
+1	NULL
+2	NULL
+3	NULL
+4	NULL
+
+statement ok
+create table t_simple_2 (v1 int, v2 int);
+
+statement ok
+create sink s_simple_2 into m_simple as select v1, v2 from t_simple_2;
+
+statement ok
+insert into t_simple_2 values (100, 101), (200, 201), (300, 301);
+
+statement ok
+flush;
+
+query II rowsort
+select * from m_simple;
+----
+1	NULL
+100	101
+2	NULL
+200	201
+3	NULL
+300	301
+4	NULL
+
+statement error dropping columns in target table of sinks is not supported
+alter table m_simple drop column v2;
+
+statement ok
+drop sink s_simple_1;
+
+statement ok
+drop sink s_simple_2;
+
+statement ok
+drop table t_simple_1;
+
+statement ok
+drop table t_simple_2;
+
+statement ok
+drop table m_simple;
+
+# target table with row_id as primary key
+statement ok
+create table t_s1 (v1 int);
+
+statement ok
+insert into t_s1 values (1), (2), (3);
+
+statement ok
+create table t_row_id_as_primary_key (v1 int, v2 int default 1000);
+
+statement ok
+create sink s1 into t_row_id_as_primary_key as select v1 from t_s1 with (type = 'append-only', force_append_only = 'true');
+
+statement ok
+flush;
+
+query II rowsort
+select * from t_row_id_as_primary_key;
+----
+1	1000
+2	1000
+3	1000
+
+statement ok
+alter table t_row_id_as_primary_key add column v3 int;
+
+query III rowsort
+select * from t_row_id_as_primary_key;
+----
+1	1000	NULL
+2	1000	NULL
+3	1000	NULL
+
+statement ok
+create sink s11 into t_row_id_as_primary_key as select v1+1000 as v1, v1+2000 as v2, v1+3000 as v3 from t_s1 with (type = 'append-only', force_append_only = 'true');
+
+statement ok
+flush;
+
+query III rowsort
+select * from t_row_id_as_primary_key;
+----
+1	1000	NULL
+1001	2001	3001
+1002	2002	3002
+1003	2003	3003
+2	1000	NULL
+3	1000	NULL
+
+statement ok
+drop sink s1;
+
+statement ok
+drop sink s11;
+
+statement ok
+drop table t_row_id_as_primary_key;
+
+statement ok
+drop table t_s1;
+
+# target table with append only
+statement ok
+create table t_s2 (v1 int);
+
+statement ok
+insert into t_s2 values (1), (2), (3);
+
+statement ok
+create table t_append_only (v1 int, v2 int default 1000) append only;
+
+statement ok
+create sink s2 into t_append_only as select v1 from t_s2 with (type = 'append-only', force_append_only = 'true');
+
+statement ok
+flush;
+
+query II rowsort
+select * from t_append_only;
+----
+1	1000
+2	1000
+3	1000
+
+statement ok
+alter table t_append_only add column v3 int;
+
+query III rowsort
+select * from t_append_only;
+----
+1	1000	NULL
+2	1000	NULL
+3	1000	NULL
+
+statement ok
+create sink s21 into t_append_only as select v1+1000 as v1, v1+2000 as v2, v1+3000 as v3 from t_s2 with (type = 'append-only', force_append_only = 'true');
+
+query III rowsort
+select * from t_append_only;
+----
+1	1000	NULL
+1001	2001	3001
+1002	2002	3002
+1003	2003	3003
+2	1000	NULL
+3	1000	NULL
+
+statement ok
+drop sink s21;
+
+statement ok
+drop sink s2;
+
+statement ok
+drop table t_append_only;
+
+statement ok
+drop table t_s2;
diff --git a/e2e_test/sink/sink_into_table/basic.slt b/e2e_test/sink/sink_into_table/basic.slt
index 59e43773560f0..e2a10d46fbf37 100644
--- a/e2e_test/sink/sink_into_table/basic.slt
+++ b/e2e_test/sink/sink_into_table/basic.slt
@@ -13,10 +13,6 @@ create table m_simple (v1 int primary key, v2 int);
 statement ok
 create sink s_simple_1 into m_simple as select v1, v2 from t_simple;
 
-# we can't alter a table with incoming sinks
-statement error Feature is not yet implemented: alter table with incoming sinks
-alter table m_simple add column v3 int;
-
 statement ok
 drop sink s_simple_1;
 
diff --git a/e2e_test/slow_tests/backfill/rate_limit/slow-udf.slt b/e2e_test/slow_tests/backfill/rate_limit/slow-udf.slt
index a2b1a6fc63130..adf133ff7f3e7 100644
--- a/e2e_test/slow_tests/backfill/rate_limit/slow-udf.slt
+++ b/e2e_test/slow_tests/backfill/rate_limit/slow-udf.slt
@@ -5,7 +5,7 @@ statement ok
 insert into t select 2 from generate_series(1, 1000000);
 
 statement ok
-set streaming_rate_limit=1;
+set backfill_rate_limit=1;
 
 statement ok
 set background_ddl=true;
@@ -25,7 +25,7 @@ statement ok
 set background_ddl = false;
 
 statement ok
-set streaming_rate_limit=default;
+set backfill_rate_limit=default;
 
 statement ok
 flush;
diff --git a/e2e_test/slow_tests/udf/always_retry_python.slt b/e2e_test/slow_tests/udf/always_retry_python.slt
index 78bf926c32986..18184846f272a 100644
--- a/e2e_test/slow_tests/udf/always_retry_python.slt
+++ b/e2e_test/slow_tests/udf/always_retry_python.slt
@@ -21,7 +21,7 @@ statement ok
 flush;
 
 statement ok
-SET STREAMING_RATE_LIMIT=1;
+SET BACKFILL_RATE_LIMIT=1;
 
 statement ok
 SET BACKGROUND_DDL=true;
@@ -57,7 +57,7 @@ SELECT count(*) FROM mv_always_retry where s1 is NULL;
 # t
 
 statement ok
-SET STREAMING_RATE_LIMIT TO DEFAULT;
+SET BACKFILL_RATE_LIMIT TO DEFAULT;
 
 statement ok
 SET BACKGROUND_DDL=false;
diff --git a/e2e_test/source/basic/alter/rate_limit_source_kafka.slt b/e2e_test/source/basic/alter/rate_limit_source_kafka.slt
index 7d991083345a1..ac56a7f48e3a5 100644
--- a/e2e_test/source/basic/alter/rate_limit_source_kafka.slt
+++ b/e2e_test/source/basic/alter/rate_limit_source_kafka.slt
@@ -31,6 +31,7 @@ create source kafka_source (v1 int) with (
   topic = 'kafka_source',
   properties.bootstrap.server = 'message_queue:29092',
   scan.startup.mode = 'earliest',
+  source_rate_limit = 0,
 ) FORMAT PLAIN ENCODE JSON
 
 statement ok
@@ -43,8 +44,9 @@ sleep 3s
 
 ############## Create MV on source
 
+# This should be ignored.
 statement ok
-SET STREAMING_RATE_LIMIT=0;
+SET SOURCE_RATE_LIMIT=1000;
 
 statement ok
 create materialized view rl_mv1 as select count(*) from kafka_source;
@@ -55,9 +57,6 @@ create materialized view rl_mv2 as select count(*) from kafka_source;
 statement ok
 create materialized view rl_mv3 as select count(*) from kafka_source;
 
-statement ok
-SET STREAMING_RATE_LIMIT=default;
-
 ############## MVs should have 0 records, since source has (rate_limit = 0)
 
 statement ok
@@ -82,11 +81,11 @@ select * from rl_mv3;
 
 skipif in-memory
 query I
-alter source kafka_source set streaming_rate_limit to 1000;
+alter source kafka_source set source_rate_limit to 1000;
 
 skipif in-memory
 query I
-alter source kafka_source set streaming_rate_limit to default;
+alter source kafka_source set source_rate_limit to default;
 
 skipif in-memory
 sleep 3s
diff --git a/e2e_test/source/basic/alter/rate_limit_table_kafka.slt b/e2e_test/source/basic/alter/rate_limit_table_kafka.slt
index bf1fd6672d6ea..1a23bbf1009de 100644
--- a/e2e_test/source/basic/alter/rate_limit_table_kafka.slt
+++ b/e2e_test/source/basic/alter/rate_limit_table_kafka.slt
@@ -13,21 +13,24 @@ create sink kafka_sink
 from
   kafka_seed_data with (
     properties.bootstrap.server = 'message_queue:29092',
-    topic = 'kafka_source',
+    topic = 'rate_limit_source_kafka_0',
     type = 'append-only',
     force_append_only='true',
     connector = 'kafka'
 );
 
+# topic may not be created yet
+sleep 4s
+
 ############## Source from kafka (rate_limit = 0)
 
 statement ok
 create table kafka_source (v1 int) with (
   connector = 'kafka',
-  topic = 'kafka_source',
+  topic = 'rate_limit_source_kafka_0',
   properties.bootstrap.server = 'message_queue:29092',
   scan.startup.mode = 'earliest',
-  streaming_rate_limit = 0
+  source_rate_limit = 0
 ) FORMAT PLAIN ENCODE JSON
 
 statement ok
@@ -61,23 +64,36 @@ select count(*) from kafka_source;
 
 skipif in-memory
 query I
-alter table kafka_source set streaming_rate_limit to 1000;
+alter table kafka_source set source_rate_limit to 1000;
+
+skipif in-memory
+sleep 3s
 
 skipif in-memory
 query I
-alter table kafka_source set streaming_rate_limit to default;
+select count(*) > 1 from kafka_source;
+----
+t
+
+############## New MV created should have rate limit = 1000.
+
+statement ok
+create materialized view mv as select * from kafka_source;
 
 skipif in-memory
 sleep 3s
 
 skipif in-memory
 query I
-select count(*) > 1 from kafka_source;
+select count(*) > 1 from mv;
 ----
 t
 
 ############## Cleanup
 
+statement ok
+drop materialized view mv;
+
 statement ok
 drop table kafka_source;
 
diff --git a/e2e_test/source/basic/ddl.slt b/e2e_test/source/basic/ddl.slt
index a56d90934c149..8e63971fb5b82 100644
--- a/e2e_test/source/basic/ddl.slt
+++ b/e2e_test/source/basic/ddl.slt
@@ -53,7 +53,7 @@ create source s (
   properties.bootstrap.server = 'message_queue:29092'
 ) FORMAT PLAIN ENCODE JSON;
 
-statement error properties `scan_startup_mode` only support earliest and latest or leave it empty
+statement error properties `scan_startup_mode` only supports earliest and latest or leaving it empty
 create source invalid_startup_mode (
   column1 varchar
 ) with (
diff --git a/e2e_test/source/basic/kafka.slt b/e2e_test/source/basic/kafka.slt
index dee5de1cbb539..40e9b46036112 100644
--- a/e2e_test/source/basic/kafka.slt
+++ b/e2e_test/source/basic/kafka.slt
@@ -391,7 +391,7 @@ create table s29 (id bytea) with (
   topic = 'kafka_source_format_bytes',
   properties.bootstrap.server = 'message_queue:29092',
   scan.startup.mode = 'earliest',
-  streaming_rate_limit = 200
+  source_rate_limit = 200
 ) FORMAT PLAIN ENCODE BYTES
 
 statement ok
diff --git a/e2e_test/source/basic/old_row_format_syntax/ddl.slt b/e2e_test/source/basic/old_row_format_syntax/ddl.slt
index b48249ca7e393..0fe67a8504b5d 100644
--- a/e2e_test/source/basic/old_row_format_syntax/ddl.slt
+++ b/e2e_test/source/basic/old_row_format_syntax/ddl.slt
@@ -7,7 +7,7 @@ create source s (
   properties.bootstrap.server = 'message_queue:29092'
 ) ROW FORMAT JSON;
 
-statement error properties `scan_startup_mode` only support earliest and latest or leave it empty
+statement error properties `scan_startup_mode` only supports earliest and latest or leaving it empty
 create source invalid_startup_mode (
   column1 varchar
 ) with (
diff --git a/e2e_test/source/cdc/cdc.check.slt b/e2e_test/source/cdc/cdc.check.slt
index 3d6ea0e179391..b0559d948c0ef 100644
--- a/e2e_test/source/cdc/cdc.check.slt
+++ b/e2e_test/source/cdc/cdc.check.slt
@@ -75,3 +75,8 @@ select * from enum_to_varchar order by id;
 ----
 1 happy
 2 ok
+
+query II
+select * from upper_orders order by id;
+----
+1 happy
diff --git a/e2e_test/source/cdc/cdc.load.slt b/e2e_test/source/cdc/cdc.load.slt
index 2c372cbd3ffdd..599394c8a0047 100644
--- a/e2e_test/source/cdc/cdc.load.slt
+++ b/e2e_test/source/cdc/cdc.load.slt
@@ -242,3 +242,19 @@ create table enum_to_varchar (
  table.name = 'enum_table',
  slot.name = 'enum_to_varchar'
 );
+
+statement ok
+create table upper_orders (
+  id int,
+  name varchar,
+  PRIMARY KEY (id)
+) with (
+ connector = 'postgres-cdc',
+ hostname = '${PGHOST:localhost}',
+ port = '${PGPORT:5432}',
+ username = '${PGUSER:$USER}',
+ password = '${PGPASSWORD:}',
+ database.name = '${PGDATABASE:postgres}',
+ table.name = 'Orders',
+ slot.name = 'orders'
+);
diff --git a/e2e_test/source/cdc/cdc.share_stream.slt b/e2e_test/source/cdc/cdc.share_stream.slt
index 3dc26d98c6282..b180a67c2cb2a 100644
--- a/e2e_test/source/cdc/cdc.share_stream.slt
+++ b/e2e_test/source/cdc/cdc.share_stream.slt
@@ -11,6 +11,11 @@ mysql --protocol=tcp -u root mytest < e2e_test/source/cdc/mysql_create.sql
 system ok
 mysql --protocol=tcp -u root mytest < e2e_test/source/cdc/mysql_init_data.sql
 
+statement ok
+create secret mysql_pwd with (
+  backend = 'meta'
+) as '${MYSQL_PWD:}';
+
 # create a cdc source job, which format fixed to `FORMAT PLAIN ENCODE JSON`
 statement ok
 create source mysql_mytest with (
@@ -18,7 +23,7 @@ create source mysql_mytest with (
  hostname = '${MYSQL_HOST:localhost}',
  port = '${MYSQL_TCP_PORT:8306}',
  username = 'rwcdc',
- password = '${MYSQL_PWD:}',
+ password = secret mysql_pwd,
  database.name = 'mytest',
  server.id = '5601'
 );
@@ -48,6 +53,9 @@ from mysql_mytest table 'mytest.products';
 # sleep to ensure (default,'Milk','Milk is a white liquid food') is consumed from Debezium message instead of backfill.
 sleep 10s
 
+statement error Permission denied
+drop secret mysql_pwd;
+
 system ok
 mysql --protocol=tcp -u root mytest -e "INSERT INTO products VALUES (default,'Milk','Milk is a white liquid food');
        INSERT INTO orders VALUES (default, '2023-11-28 15:08:22', 'Bob', 10.52, 100, false);"
@@ -190,13 +198,23 @@ SELECT c_tinyint, c_smallint, c_mediumint, c_integer, c_bigint, c_decimal, c_flo
 -128 -32767 -8388608 -2147483647 -9223372036854775807 -10 -10000 -10000 a b 1001-01-01 00:00:00 1998-01-01 00:00:00 1970-01-01 00:00:01+00:00
 NULL NULL -8388608 -2147483647 9223372036854775806 -10 -10000 -10000 c d 1001-01-01 NULL 2000-01-01 00:00:00 NULL
 
+statement ok
+create secret pg_pwd with (
+  backend = 'meta'
+) as '${PGPASSWORD:}';
+
+statement ok
+create secret pg_username with (
+  backend = 'meta'
+) as '${PGUSER:$USER}';
+
 statement ok
 create source pg_source with (
  connector = 'postgres-cdc',
  hostname = '${PGHOST:localhost}',
  port = '${PGPORT:5432}',
- username = '${PGUSER:$USER}',
- password = '${PGPASSWORD:}',
+ username = secret pg_username,
+ password = secret pg_pwd,
  database.name = '${PGDATABASE:postgres}',
  slot.name = 'pg_slot'
 );
@@ -250,6 +268,12 @@ CREATE TABLE person_new (
     city varchar
 ) FROM pg_source TABLE 'person';
 
+statement ok
+CREATE TABLE upper_orders_shared (
+    id int PRIMARY KEY,
+    name varchar
+) FROM pg_source TABLE 'public.Orders';
+
 statement ok
 CREATE TABLE person_new (
     id int,
@@ -491,3 +515,8 @@ select * from enum_to_varchar_shared order by id;
 ----
 1 happy
 2 ok
+
+query II
+select * from upper_orders_shared order by id;
+----
+1 happy
diff --git a/e2e_test/source/cdc/postgres_cdc.sql b/e2e_test/source/cdc/postgres_cdc.sql
index 6579bc2683037..20b6e7b414469 100644
--- a/e2e_test/source/cdc/postgres_cdc.sql
+++ b/e2e_test/source/cdc/postgres_cdc.sql
@@ -1,4 +1,5 @@
 -- PG
+DROP TABLE IF EXISTS shipments;
 CREATE TABLE shipments (
   shipment_id SERIAL NOT NULL PRIMARY KEY,
   order_id SERIAL NOT NULL,
@@ -105,3 +106,9 @@ CREATE TABLE list_with_null(id int primary key, my_int int[], my_num numeric[],
 INSERT INTO list_with_null VALUES (1, '{1,2,NULL}', '{1.1,inf,NULL}', '{1.1,inf,NULL}', '{1.1,inf,NULL}', '{happy,ok,NULL}', '{bb488f9b-330d-4012-b849-12adeb49e57e,bb488f9b-330d-4012-b849-12adeb49e57f, NULL}', '{\\x00,\\x01,NULL}');
 INSERT INTO list_with_null VALUES (2, '{NULL,3,4}', '{2.2,0,NULL}' , '{2.2,0,NULL}', '{2.2,0,NULL}', '{happy,ok,sad}', '{2de296df-eda7-4202-a81f-1036100ef4f6,2977afbc-0b12-459c-a36f-f623fc9e9840}', '{\\x00,\\x01,\\x02}');
 INSERT INTO list_with_null VALUES (5, NULL, NULL, NULL, NULL, NULL, NULL, NULL);
+
+CREATE TABLE "Orders" (
+    id int PRIMARY KEY,
+    name varchar
+);
+INSERT INTO "Orders" VALUES (1, 'happy');
diff --git a/e2e_test/source/cdc_inline/alter/cdc_table_alter.slt b/e2e_test/source/cdc_inline/alter/cdc_table_alter.slt
new file mode 100644
index 0000000000000..6bea5dce2fe45
--- /dev/null
+++ b/e2e_test/source/cdc_inline/alter/cdc_table_alter.slt
@@ -0,0 +1,242 @@
+control substitution on
+
+# mysql env vars will be read from the `.risingwave/config/risedev-env` file
+
+system ok
+mysql -e "
+    SET GLOBAL time_zone = '+00:00';
+"
+
+system ok
+mysql -e "
+  DROP DATABASE IF EXISTS testdb1;
+  CREATE DATABASE testdb1;
+  USE testdb1;
+  CREATE TABLE products (
+      id INTEGER NOT NULL AUTO_INCREMENT PRIMARY KEY,
+      name VARCHAR(255) NOT NULL,
+      description VARCHAR(512)
+  );
+  ALTER TABLE products AUTO_INCREMENT = 101;
+  INSERT INTO products
+  VALUES (default,'scooter','Small 2-wheel scooter'),
+         (default,'car battery','12V car battery'),
+         (default,'12-pack drill','12-pack of drill bits with sizes ranging from #40 to #3'),
+         (default,'hammer','12oz carpenter s hammer'),
+         (default,'hammer','14oz carpenter s hammer'),
+         (default,'hammer','16oz carpenter s hammer'),
+         (default,'rocks','box of assorted rocks'),
+         (default,'jacket','water resistent black wind breaker'),
+         (default,'spare tire','24 inch spare tire');
+  CREATE TABLE orders (
+      order_id INTEGER NOT NULL AUTO_INCREMENT PRIMARY KEY,
+      order_date DATETIME NOT NULL,
+      customer_name VARCHAR(255) NOT NULL,
+      price DECIMAL(10, 5) NOT NULL,
+      product_id INTEGER NOT NULL,
+      order_status BOOLEAN NOT NULL
+  ) AUTO_INCREMENT = 10001;
+  INSERT INTO orders
+  VALUES (default, '2020-07-30 10:08:22', 'Jark', 50.50, 102, false),
+         (default, '2020-07-30 10:11:09', 'Sally', 15.00, 105, false),
+         (default, '2020-07-30 12:00:30', 'Edward', 25.25, 106, false);
+"
+
+statement ok
+create source mysql_source with (
+  connector = 'mysql-cdc',
+  hostname = '${MYSQL_HOST}',
+  port = '${MYSQL_TCP_PORT}',
+  username = 'root',
+  password = '${MYSQL_PWD}',
+  database.name = 'testdb1',
+  server.id = '5185'
+);
+
+statement ok
+create table my_products ( id INT,
+ name STRING,
+ description STRING,
+ PRIMARY KEY (id)
+) from mysql_source table 'testdb1.products';
+
+statement ok
+create table my_orders (
+   order_id int,
+   order_date timestamp,
+   customer_name string,
+   price decimal,
+   product_id int,
+   order_status smallint,
+   PRIMARY KEY (order_id)
+) from mysql_source table 'testdb1.orders';
+
+system ok
+psql -c "
+  DROP TABLE IF EXISTS shipments1;
+  CREATE TABLE shipments1 (
+    shipment_id SERIAL NOT NULL PRIMARY KEY,
+    order_id SERIAL NOT NULL,
+    origin VARCHAR(255) NOT NULL,
+    destination VARCHAR(255) NOT NULL,
+    is_arrived BOOLEAN NOT NULL
+  );
+  ALTER SEQUENCE public.shipments1_shipment_id_seq RESTART WITH 1001;
+  INSERT INTO shipments1
+  VALUES (default,10001,'Beijing','Shanghai',false),
+         (default,10002,'Hangzhou','Shanghai',false),
+         (default,10003,'Shanghai','Hangzhou',false);
+"
+
+statement ok
+create source pg_source with (
+  connector = 'postgres-cdc',
+  hostname = '${PGHOST:localhost}',
+  port = '${PGPORT:5432}',
+  username = '${PGUSER:$USER}',
+  password = '${PGPASSWORD:}',
+  database.name = '${PGDATABASE:postgres}',
+  slot.name = 'cdc_alter_test'
+);
+
+statement ok
+create table pg_shipments (
+  shipment_id INTEGER,
+  order_id INTEGER,
+  origin STRING,
+  destination STRING,
+  is_arrived boolean,
+ PRIMARY KEY (shipment_id)
+) from pg_source table 'public.shipments1';
+
+# Create a mview join orders, products and shipments
+statement ok
+create materialized view enriched_orders as SELECT o.*, p.name, p.description, s.shipment_id, s.origin, s.destination, s.is_arrived
+ FROM my_orders AS o
+ LEFT JOIN my_products AS p ON o.product_id = p.id
+ LEFT JOIN pg_shipments AS s ON o.order_id = s.order_id;
+
+
+sleep 3s
+
+query III
+select order_id, product_id, shipment_id from enriched_orders order by order_id;
+----
+10001  102   1001
+10002  105   1002
+10003  106   1003
+
+
+# alter mysql tables
+system ok
+mysql -e "
+  USE testdb1;
+  ALTER TABLE products ADD COLUMN weight DECIMAL(10, 2) NOT NULL DEFAULT 0.0;
+  ALTER TABLE orders ADD COLUMN order_comment VARCHAR(255);
+"
+
+# alter cdc tables
+statement ok
+ALTER TABLE my_products ADD COLUMN weight DECIMAL;
+
+statement ok
+ALTER TABLE my_orders ADD COLUMN order_comment VARCHAR;
+
+# wait alter ddl
+sleep 3s
+
+query ITTT
+SELECT id,name,description,weight FROM my_products order by id limit 3
+----
+101 scooter Small 2-wheel scooter NULL
+102 car battery 12V car battery NULL
+103 12-pack drill 12-pack of drill bits with sizes ranging from #40 to #3 NULL
+
+
+# update mysql tables
+system ok
+mysql -e "
+  USE testdb1;
+  UPDATE products SET weight = 10.5 WHERE id = 101;
+  UPDATE products SET weight = 12.5 WHERE id = 102;
+  UPDATE orders SET order_comment = 'very good' WHERE order_id = 10001;
+"
+
+sleep 3s
+
+query ITTT
+SELECT id,name,description,weight FROM my_products order by id limit 3
+----
+101 scooter Small 2-wheel scooter 10.50
+102 car battery 12V car battery 12.50
+103 12-pack drill 12-pack of drill bits with sizes ranging from #40 to #3 NULL
+
+query ITTT
+SELECT order_id,order_date,customer_name,product_id,order_status,order_comment FROM my_orders order by order_id limit 2
+----
+10001 2020-07-30 10:08:22 Jark 102 0 very good
+10002 2020-07-30 10:11:09 Sally 105 0 NULL
+
+
+# alter mysql tables
+system ok
+mysql -e "
+  USE testdb1;
+  ALTER TABLE products DROP COLUMN weight;
+"
+
+# alter cdc table to drop column
+statement ok
+ALTER TABLE my_products DROP COLUMN weight;
+
+# wait alter ddl
+sleep 3s
+
+query TTTT
+describe my_products;
+----
+id integer false NULL
+name character varying false NULL
+description character varying false NULL
+primary key id NULL NULL
+distribution key id NULL NULL
+table description my_products NULL NULL
+
+
+# alter pg table
+system ok
+psql -c "
+  ALTER TABLE shipments1 DROP COLUMN destination;
+"
+
+statement error unable to drop the column due to being referenced by downstream materialized views or sinks
+ALTER TABLE pg_shipments DROP COLUMN destination;
+
+# wait alter ddl
+sleep 3s
+
+# query mv again
+query III
+select order_id, product_id, shipment_id from enriched_orders order by order_id;
+----
+10001  102   1001
+10002  105   1002
+10003  106   1003
+
+statement ok
+drop materialized view enriched_orders;
+
+statement ok
+drop table my_orders;
+
+statement ok
+create table orders_test (*) from mysql_source table 'testdb1.orders';
+
+statement error Not supported: alter a table with empty column definitions
+ALTER TABLE orders_test ADD COLUMN order_comment VARCHAR;
+
+statement ok
+drop source mysql_source cascade;
+
+statement ok
+drop source pg_source cascade;
diff --git a/e2e_test/source/cdc_inline/auto_schema_map_mysql.slt b/e2e_test/source/cdc_inline/auto_schema_map_mysql.slt
index eaa727b610828..b1a4cd0836349 100644
--- a/e2e_test/source/cdc_inline/auto_schema_map_mysql.slt
+++ b/e2e_test/source/cdc_inline/auto_schema_map_mysql.slt
@@ -40,6 +40,9 @@ mysql --protocol=tcp -u root mytest -e "
     INSERT INTO mysql_types_test VALUES ( True, 1, -128, -32767, -8388608, -2147483647, -9223372036854775807, -10.0, -9999.999999, -10000.0, 'a', 'b', '', '', '1001-01-01', '00:00:00', '1998-01-01 00:00:00.000000', '1970-01-01 00:00:01', 'sad', '[3,4]');
     "
 
+statement ok
+ALTER SYSTEM SET license_key TO '';
+
 statement ok
 create source mysql_source with (
  connector = 'mysql-cdc',
@@ -51,6 +54,22 @@ create source mysql_source with (
  server.id = '5601'
 );
 
+statement error
+create table rw_customers (*) from mysql_source table 'mytest.customers';
+----
+db error: ERROR: Failed to run the query
+
+Caused by:
+  Not supported: feature CdcTableSchemaMap is only available for tier Paid and above, while the current tier is Free
+
+Hint: You may want to set a license key with `ALTER SYSTEM SET license_key = '...';` command.
+HINT: Please define the schema manually
+
+
+
+statement ok
+ALTER SYSTEM SET license_key TO DEFAULT;
+
 statement ok
 create table rw_customers (*) from mysql_source table 'mytest.customers';
 
diff --git a/e2e_test/source/cdc_inline/sql_server_cdc/sql_server_cdc.slt b/e2e_test/source/cdc_inline/sql_server_cdc/sql_server_cdc.slt
new file mode 100644
index 0000000000000..ec62c3d08adf4
--- /dev/null
+++ b/e2e_test/source/cdc_inline/sql_server_cdc/sql_server_cdc.slt
@@ -0,0 +1,364 @@
+
+control substitution on
+
+# ------------ data prepare stage ------------
+system ok
+sqlcmd -C -d master -Q 'create database mydb;' -b
+
+system ok
+sqlcmd -C -i e2e_test/source/cdc_inline/sql_server_cdc/sql_server_cdc_prepare.sql -b
+
+# ------------ validate stage ------------
+
+# invalid address, comment this test out because it takes long to wait for TCP connection timeout.
+# statement error The TCP/IP connection to the host wrong-sqlserver-server, port 1433 has failed.
+# CREATE SOURCE mssql_source WITH (
+#     connector = 'sqlserver-cdc',
+#     hostname = 'wrong-sqlserver-server',
+#     port = '${SQLCMDPORT:1433}',
+#     username = '${SQLCMDUSER:SA}',
+#     password = '${SQLCMDPASSWORD}',
+#     database.name = '${SQLCMDDBNAME}',
+# );
+
+# invalid username
+statement error Login failed for user 'SAP'
+CREATE SOURCE mssql_source WITH (
+    connector = 'sqlserver-cdc',
+    hostname = '${SQLCMDSERVER:sqlserver-server}',
+    port = '${SQLCMDPORT:1433}',
+    username = 'SAP',
+    password = '${SQLCMDPASSWORD}',
+    database.name = '${SQLCMDDBNAME}',
+);
+
+# invalid password
+statement error Login failed for user
+CREATE SOURCE mssql_source WITH (
+    connector = 'sqlserver-cdc',
+    hostname = '${SQLCMDSERVER:sqlserver-server}',
+    port = '${SQLCMDPORT:1433}',
+    username = '${SQLCMDUSER:SA}',
+    password = 'MyPassword123',
+    database.name = '${SQLCMDDBNAME}',
+);
+
+# invalid database
+statement error Cannot open database "wrong_mydb" requested by the login. The login failed.
+CREATE SOURCE mssql_source WITH (
+    connector = 'sqlserver-cdc',
+    hostname = '${SQLCMDSERVER:sqlserver-server}',
+    port = '${SQLCMDPORT:1433}',
+    username = '${SQLCMDUSER:SA}',
+    password = '${SQLCMDPASSWORD}',
+    database.name = 'wrong_mydb',
+);
+
+statement error Protocol error: connector sqlserver-cdc does not support `CREATE TABLE`, please use `CREATE SOURCE` instead
+CREATE TABLE orders (
+  order_id INT PRIMARY KEY,
+  order_date BIGINT,
+  customer_name VARCHAR,
+  price DECIMAL,
+  product_id INT,
+  order_status SMALLINT
+) WITH (
+  connector = 'sqlserver-cdc',
+  hostname = '${SQLCMDSERVER:sqlserver-server}',
+  port = '${SQLCMDPORT:1433}',
+  username = '${SQLCMDUSER:SA}',
+  password = '${SQLCMDPASSWORD}',
+  table.name = 'orders',
+  database.name = '${SQLCMDDBNAME}',
+);
+
+statement error Protocol error: connector sqlserver-cdc does not support `CREATE TABLE`, please use `CREATE SOURCE` instead
+CREATE TABLE single_type (
+  id INT,
+  c_time time,
+  PRIMARY KEY (id)
+)  WITH (
+  connector = 'sqlserver-cdc',
+  hostname = '${SQLCMDSERVER:sqlserver-server}',
+  port = '${SQLCMDPORT:1433}',
+  username = '${SQLCMDUSER:SA}',
+  password = '${SQLCMDPASSWORD}',
+  table.name = 'single_type',
+  database.name = '${SQLCMDDBNAME}',
+);
+
+statement error Protocol error: connector sqlserver-cdc does not support `CREATE TABLE`, please use `CREATE SOURCE` instead
+CREATE TABLE sqlserver_all_data_types (
+  id INT PRIMARY KEY,
+  c_bit BOOLEAN,
+  c_tinyint SMALLINT,
+  c_smallint SMALLINT,
+  c_int INTEGER,
+  c_bigint BIGINT,
+  c_decimal DECIMAL,
+  c_real REAL,
+  c_float FLOAT,
+  c_varchar VARCHAR,
+  c_varbinary BYTEA,
+  c_date DATE,
+  c_time TIME,
+  c_datetime2 TIMESTAMP,
+  c_datetimeoffset TIMESTAMPTZ
+) WITH (
+  connector = 'sqlserver-cdc',
+  hostname = '${SQLCMDSERVER:sqlserver-server}',
+  port = '${SQLCMDPORT:1433}',
+  username = '${SQLCMDUSER:SA}',
+  password = '${SQLCMDPASSWORD}',
+  table.name = 'sqlserver_all_data_types',
+  database.name = '${SQLCMDDBNAME}',
+);
+
+# ------------ Create source/table/mv stage ------------
+# create a cdc source job, which format fixed to `FORMAT PLAIN ENCODE JSON`
+statement ok
+CREATE SOURCE mssql_source WITH (
+    connector = 'sqlserver-cdc',
+    hostname = '${SQLCMDSERVER:sqlserver-server}',
+    port = '${SQLCMDPORT:1433}',
+    username = '${SQLCMDUSER:SA}',
+    password = '${SQLCMDPASSWORD}',
+    database.name = '${SQLCMDDBNAME}',
+);
+
+statement error Should not create MATERIALIZED VIEW or SELECT directly on shared CDC source
+create materialized view mv as select * from mssql_source;
+
+statement error The upstream table name must contain schema name prefix*
+CREATE TABLE shared_orders (
+    order_id INT,
+    order_date BIGINT,
+    customer_name VARCHAR,
+    price DECIMAL,
+    product_id INT,
+    order_status SMALLINT,
+    PRIMARY KEY (order_id)
+)  from mssql_source table 'orders';
+
+# invalid table name
+statement error Sql Server table 'dbo'.'wrong_orders' doesn't exist
+CREATE TABLE shared_orders (
+    order_id INT,
+    order_date BIGINT,
+    customer_name VARCHAR,
+    price DECIMAL,
+    product_id INT,
+    order_status SMALLINT,
+    PRIMARY KEY (order_id)
+)  from mssql_source table 'dbo.wrong_orders';
+
+# invalid schema name
+statement error Sql Server table 'wrong_dbo'.'orders' doesn't exist
+CREATE TABLE shared_orders (
+    order_id INT,
+    order_date BIGINT,
+    customer_name VARCHAR,
+    price DECIMAL,
+    product_id INT,
+    order_status SMALLINT,
+    PRIMARY KEY (order_id)
+)  from mssql_source table 'wrong_dbo.orders';
+
+# invalid primary key
+statement error INVALID_ARGUMENT: Primary key mismatch
+CREATE TABLE shared_orders (
+    order_id INT,
+    order_date BIGINT,
+    customer_name VARCHAR,
+    price DECIMAL,
+    product_id INT,
+    order_status SMALLINT,
+    PRIMARY KEY (order_date)
+)  from mssql_source table 'dbo.orders';
+
+# column name mismatch
+statement error INVALID_ARGUMENT: Column 'wrong_order_date' not found in the upstream database
+CREATE TABLE shared_orders (
+    order_id INT,
+    wrong_order_date BIGINT,
+    customer_name VARCHAR,
+    price DECIMAL,
+    product_id INT,
+    order_status SMALLINT,
+    PRIMARY KEY (order_id)
+)  from mssql_source table 'dbo.orders';
+
+# column data type mismatch
+statement error INVALID_ARGUMENT: Incompatible data type of column order_date
+CREATE TABLE shared_orders (
+    order_id INT,
+    order_date VARCHAR,
+    customer_name VARCHAR,
+    price DECIMAL,
+    product_id INT,
+    order_status SMALLINT,
+    PRIMARY KEY (order_id)
+)  from mssql_source table 'dbo.orders';
+
+# table without enabling cdc
+statement error INVALID_ARGUMENT: Table 'dbo.orders_without_cdc' has not enabled CDC.
+CREATE TABLE shared_orders_without_cdc (
+    order_id INT,
+    order_date BIGINT,
+    customer_name VARCHAR,
+    price DECIMAL,
+    product_id INT,
+    order_status SMALLINT,
+    PRIMARY KEY (order_id)
+)  from mssql_source table 'dbo.orders_without_cdc';
+
+statement ok
+CREATE TABLE shared_orders (
+    order_id INT,
+    order_date BIGINT,
+    customer_name VARCHAR,
+    price DECIMAL,
+    product_id INT,
+    order_status SMALLINT,
+    PRIMARY KEY (order_id)
+)  from mssql_source table 'dbo.orders';
+
+statement ok
+CREATE TABLE shared_single_type (
+    id INT,
+    c_time time,
+    PRIMARY KEY (id)
+) from mssql_source table 'dbo.single_type';
+
+statement ok
+CREATE TABLE shared_sqlserver_all_data_types (
+    id INT,
+    c_bit BOOLEAN,
+    c_tinyint SMALLINT,
+    c_smallint SMALLINT,
+    c_int INTEGER,
+    c_bigint BIGINT,
+    c_decimal DECIMAL,
+    c_real REAL,
+    c_float FLOAT,
+    c_varchar VARCHAR,
+    c_varbinary BYTEA,
+    c_date DATE,
+    c_time TIME,
+    c_datetime2 TIMESTAMP,
+    c_datetimeoffset TIMESTAMPTZ,
+    PRIMARY KEY (id)
+) from mssql_source table 'dbo.sqlserver_all_data_types';
+
+statement ok
+create materialized view shared_orders_cnt as select count(*) as cnt from shared_orders;
+
+statement ok
+create materialized view shared_single_type_cnt as select count(*) as cnt from shared_single_type;
+
+statement ok
+create materialized view shared_sqlserver_all_data_types_cnt as select count(*) as cnt from shared_sqlserver_all_data_types;
+
+# sleep to ensure the data in mssql tables is consumed from Debezium message instead of backfill.
+sleep 20s
+
+# ------------ check stage ------------
+query I
+select cnt from shared_orders_cnt;
+----
+3
+
+query I
+select cnt from shared_single_type_cnt;
+----
+1
+
+query I
+select cnt from shared_sqlserver_all_data_types_cnt;
+----
+3
+
+query III
+select * from shared_orders order by order_id;
+----
+1 1558430840000 Bob 11 1 1
+2 1558430840001 Alice 21 2 1
+3 1558430840002 Alice 19 2 1
+
+query I
+SELECT * from shared_single_type order by id;
+----
+3 23:59:59.999
+
+query TTTTTTT
+SELECT * from shared_sqlserver_all_data_types order by id;
+----
+1 f 0 0 0 0 0 0 0 (empty) NULL 2001-01-01 00:00:00 2001-01-01 00:00:00 2001-01-01 00:00:00+00:00
+2 t 255 -32768 -2147483648 -9223372036854775808 -10 -10000 -10000 aa \xff 1990-01-01 13:59:59.123 2000-01-01 11:00:00.123 1990-01-01 00:00:01.123+00:00
+3 t 127 32767 2147483647 9223372036854775807 -10 10000 10000 zzzz \xffffffff 2999-12-31 23:59:59.999 2099-12-31 23:59:59.999 2999-12-31 23:59:59.999+00:00
+
+# ------------ kill cluster ------------
+# system ok
+# risedev kill
+
+# sleep 30s
+
+# ------------ add rows stage ------------
+system ok
+sqlcmd -C -i e2e_test/source/cdc_inline/sql_server_cdc/sql_server_cdc_insert.sql -b
+
+sleep 10s
+
+# ------------ recover cluster ------------
+# system ok
+# risedev dev ci-1cn-1fe-with-recovery
+
+# sleep 30s
+
+# ------------ check after recovery stage ------------
+
+query I
+select cnt from shared_orders_cnt;
+----
+6
+
+query I
+select cnt from shared_single_type_cnt;
+----
+2
+
+query I
+select cnt from shared_sqlserver_all_data_types_cnt;
+----
+6
+
+
+query III
+select * from shared_orders order by order_id;
+----
+1 1558430840000 Bob 11 1 1
+2 1558430840001 Alice 21 2 1
+3 1558430840002 Alice 19 2 1
+11 1558430840000 Bob 11 1 1
+12 1558430840001 Alice 21 2 1
+13 1558430840002 Alice 19 2 1
+
+query I
+SELECT * from shared_single_type order by id;
+----
+3 23:59:59.999
+13 23:59:59.999
+
+query TTTTTTT
+SELECT * from shared_sqlserver_all_data_types order by id;
+----
+1 f 0 0 0 0 0 0 0 (empty) NULL 2001-01-01 00:00:00 2001-01-01 00:00:00 2001-01-01 00:00:00+00:00
+2 t 255 -32768 -2147483648 -9223372036854775808 -10 -10000 -10000 aa \xff 1990-01-01 13:59:59.123 2000-01-01 11:00:00.123 1990-01-01 00:00:01.123+00:00
+3 t 127 32767 2147483647 9223372036854775807 -10 10000 10000 zzzz \xffffffff 2999-12-31 23:59:59.999 2099-12-31 23:59:59.999 2999-12-31 23:59:59.999+00:00
+11 f 0 0 0 0 0 0 0 (empty) NULL 2001-01-01 00:00:00 2001-01-01 00:00:00 2001-01-01 00:00:00+00:00
+12 t 255 -32768 -2147483648 -9223372036854775808 -10 -10000 -10000 aa \xff 1990-01-01 13:59:59.123 2000-01-01 11:00:00.123 1990-01-01 00:00:01.123+00:00
+13 t 127 32767 2147483647 9223372036854775807 -10 10000 10000 zzzz \xffffffff 2999-12-31 23:59:59.999 2099-12-31 23:59:59.999 2999-12-31 23:59:59.999+00:00
+
+# ------------ drop stage ------------
+statement ok
+drop source mssql_source cascade;
diff --git a/e2e_test/source/cdc_inline/sql_server_cdc/sql_server_cdc_insert.sql b/e2e_test/source/cdc_inline/sql_server_cdc/sql_server_cdc_insert.sql
new file mode 100644
index 0000000000000..e5089aa86d020
--- /dev/null
+++ b/e2e_test/source/cdc_inline/sql_server_cdc/sql_server_cdc_insert.sql
@@ -0,0 +1,24 @@
+
+INSERT INTO
+  orders (
+    order_id,
+    order_date,
+    customer_name,
+    price,
+    product_id,
+    order_status
+  )
+VALUES
+  (11, 1558430840000, 'Bob', 10.50, 1, 1),
+  (12, 1558430840001, 'Alice', 20.50, 2, 1),
+  (13, 1558430840002, 'Alice', 18.50, 2, 1);
+
+
+INSERT INTO single_type VALUES (13, '23:59:59.999')
+
+
+INSERT INTO sqlserver_all_data_types VALUES (11, 'False', 0, 0, 0, 0, 0, 0, 0, '', NULL, '2001-01-01', '00:00:00', '2001-01-01 00:00:00', '2001-01-01 00:00:00');
+
+INSERT INTO sqlserver_all_data_types VALUES (12, 'True', 255, -32768, -2147483648, -9223372036854775808, -10.0, -9999.999999, -10000.0, 'aa', 0xff, '1990-01-01', '13:59:59.123', '2000-01-01 11:00:00.123', '1990-01-01 00:00:01.123');
+
+INSERT INTO sqlserver_all_data_types VALUES (13, 'True', 127, 32767, 2147483647, 9223372036854775807, -10.0, 9999.999999, 10000.0, 'zzzz', 0xffffffff, '2999-12-31', '23:59:59.999', '2099-12-31 23:59:59.999', '2999-12-31 23:59:59.999')
diff --git a/e2e_test/source/cdc_inline/sql_server_cdc/sql_server_cdc_prepare.sql b/e2e_test/source/cdc_inline/sql_server_cdc/sql_server_cdc_prepare.sql
new file mode 100644
index 0000000000000..792f1ddae1034
--- /dev/null
+++ b/e2e_test/source/cdc_inline/sql_server_cdc/sql_server_cdc_prepare.sql
@@ -0,0 +1,82 @@
+EXEC sys.sp_cdc_enable_db;
+
+CREATE TABLE orders (
+  order_id INT PRIMARY KEY,
+  order_date BIGINT,
+  customer_name NVARCHAR(200),
+  price DECIMAL,
+  product_id INT,
+  order_status SMALLINT
+);
+
+EXEC sys.sp_cdc_enable_table
+  @source_schema = 'dbo',
+  @source_name = 'orders',
+  @role_name = NULL;
+
+
+INSERT INTO
+  orders (
+    order_id,
+    order_date,
+    customer_name,
+    price,
+    product_id,
+    order_status
+  )
+VALUES
+  (1, 1558430840000, 'Bob', 10.50, 1, 1),
+  (2, 1558430840001, 'Alice', 20.50, 2, 1),
+  (3, 1558430840002, 'Alice', 18.50, 2, 1);
+
+CREATE TABLE single_type (
+  id INT PRIMARY KEY,
+  c_time time,
+);
+
+EXEC sys.sp_cdc_enable_table
+  @source_schema = 'dbo',
+  @source_name = 'single_type',
+  @role_name = NULL;
+
+INSERT INTO single_type VALUES (3, '23:59:59.999')
+
+
+CREATE TABLE sqlserver_all_data_types (
+  id INT PRIMARY KEY,
+  c_bit bit,
+  c_tinyint tinyint,
+  c_smallint smallint,
+  c_int int,
+  c_bigint bigint,
+  c_decimal DECIMAL(28),
+  c_real real,
+  c_float float,
+  c_varchar varchar(4),
+  c_varbinary varbinary(4),
+  c_date date,
+  c_time time,
+  c_datetime2 datetime2,
+  c_datetimeoffset datetimeoffset
+);
+
+EXEC sys.sp_cdc_enable_table
+  @source_schema = 'dbo',
+  @source_name = 'sqlserver_all_data_types',
+  @role_name = NULL;
+
+INSERT INTO sqlserver_all_data_types VALUES (1, 'False', 0, 0, 0, 0, 0, 0, 0, '', NULL, '2001-01-01', '00:00:00', '2001-01-01 00:00:00', '2001-01-01 00:00:00');
+
+INSERT INTO sqlserver_all_data_types VALUES (2, 'True', 255, -32768, -2147483648, -9223372036854775808, -10.0, -9999.999999, -10000.0, 'aa', 0xff, '1990-01-01', '13:59:59.123', '2000-01-01 11:00:00.123', '1990-01-01 00:00:01.123');
+
+INSERT INTO sqlserver_all_data_types VALUES (3, 'True', 127, 32767, 2147483647, 9223372036854775807, -10.0, 9999.999999, 10000.0, 'zzzz', 0xffffffff, '2999-12-31', '23:59:59.999', '2099-12-31 23:59:59.999', '2999-12-31 23:59:59.999')
+
+-- Table without enabling CDC
+CREATE TABLE orders_without_cdc (
+  order_id INT PRIMARY KEY,
+  order_date BIGINT,
+  customer_name NVARCHAR(200),
+  price DECIMAL,
+  product_id INT,
+  order_status SMALLINT
+);
diff --git a/e2e_test/source_inline/kafka/avro/alter_source.slt b/e2e_test/source_inline/kafka/avro/alter_source.slt
index 446fc6196d32b..57677af57cd92 100644
--- a/e2e_test/source_inline/kafka/avro/alter_source.slt
+++ b/e2e_test/source_inline/kafka/avro/alter_source.slt
@@ -12,8 +12,7 @@ system ok
 rpk topic create 'avro_alter_source_test'
 
 system ok
-echo '{"type":"record","name":"Root","fields":[{"name":"foo","type":"string"}]}' | jq '{"schema": tojson}' \
-| curl -X POST -H 'content-type:application/json' -d @- "${RISEDEV_SCHEMA_REGISTRY_URL}/subjects/avro_alter_source_test-value/versions"
+sr_register avro_alter_source_test-value '{"type":"record","name":"Root","fields":[{"name":"foo","type":"string"}]}'
 
 statement ok
 create source s
@@ -27,8 +26,7 @@ FORMAT PLAIN ENCODE AVRO (
 
 # create a new version of schema and produce a message
 system ok
-echo '{"type":"record","name":"Root","fields":[{"name":"bar","type":"int","default":0},{"name":"foo","type":"string"}]}' | jq '{"schema": tojson}' \
-| curl -X POST -H 'content-type:application/json' -d @- "${RISEDEV_SCHEMA_REGISTRY_URL}/subjects/avro_alter_source_test-value/versions"
+sr_register avro_alter_source_test-value '{"type":"record","name":"Root","fields":[{"name":"bar","type":"int","default":0},{"name":"foo","type":"string"}]}'
 
 system ok
 echo '{"foo":"ABC", "bar":1}' | rpk topic produce --schema-id=topic avro_alter_source_test
diff --git a/e2e_test/source_inline/kafka/avro/alter_table.slt b/e2e_test/source_inline/kafka/avro/alter_table.slt
new file mode 100644
index 0000000000000..e8c43739b4746
--- /dev/null
+++ b/e2e_test/source_inline/kafka/avro/alter_table.slt
@@ -0,0 +1,82 @@
+control substitution on
+
+# https://github.com/risingwavelabs/risingwave/issues/16486
+
+# cleanup
+system ok
+rpk topic delete 'avro_alter_table_test' || true; \
+(rpk sr subject delete 'avro_alter_table_test-value' && rpk sr subject delete 'avro_alter_table_test-value' --permanent) || true;
+
+# create topic and sr subject
+system ok
+rpk topic create 'avro_alter_table_test'
+
+# create a schema and produce a message
+system ok
+echo '{"type":"record","name":"Root","fields":[{"name":"bar","type":"int","default":0},{"name":"foo","type":"string"}]}' | jq '{"schema": tojson}' \
+| curl -s -X POST -H 'content-type:application/json' -d @- "${RISEDEV_SCHEMA_REGISTRY_URL}/subjects/avro_alter_table_test-value/versions"
+
+system ok
+echo '{"foo":"ABC", "bar":1}' | rpk topic produce --schema-id=topic avro_alter_table_test
+
+statement ok
+create table t (*, gen_col int as bar + 1)
+WITH (
+    ${RISEDEV_KAFKA_WITH_OPTIONS_COMMON},
+    topic = 'avro_alter_table_test'
+)
+FORMAT PLAIN ENCODE AVRO (
+    schema.registry = '${RISEDEV_SCHEMA_REGISTRY_URL}'
+);
+
+sleep 4s
+
+query ?
+select * from t
+----
+1 ABC 2
+
+# create a new version of schema that removed field bar
+system ok
+echo '{"type":"record","name":"Root","fields":[{"name":"foo","type":"string"}]}' | jq '{"schema": tojson}' \
+| curl -s -X POST -H 'content-type:application/json' -d @- "${RISEDEV_SCHEMA_REGISTRY_URL}/subjects/avro_alter_table_test-value/versions"
+
+# Refresh table schema should fail
+statement error
+ALTER TABLE t REFRESH SCHEMA;
+----
+db error: ERROR: Failed to run the query
+
+Caused by these errors (recent errors listed first):
+  1: failed to refresh schema because some of the columns to drop are referenced by a generated column "gen_col"
+  2: fail to bind expression in generated column "gen_col"
+  3: Failed to bind expression: bar + 1
+  4: Item not found: Invalid column: bar
+
+
+# Can't drop non-generated column
+statement error
+ALTER TABLE t DROP COLUMN foo;
+----
+db error: ERROR: Failed to run the query
+
+Caused by:
+  Not supported: alter table with schema registry
+HINT: try `ALTER TABLE .. FORMAT .. ENCODE .. (...)` instead
+
+
+# Drop generated column
+statement ok
+ALTER TABLE t DROP COLUMN gen_col;
+
+# Refresh table schema
+statement ok
+ALTER TABLE t REFRESH SCHEMA;
+
+query ?
+select * from t
+----
+ABC
+
+statement ok
+drop table t;
diff --git a/e2e_test/source_inline/kafka/avro/glue.slt b/e2e_test/source_inline/kafka/avro/glue.slt
new file mode 100644
index 0000000000000..4be9378b530df
--- /dev/null
+++ b/e2e_test/source_inline/kafka/avro/glue.slt
@@ -0,0 +1,148 @@
+control substitution on
+
+system ok
+rpk topic delete 'glue-sample-my-event'
+
+system ok
+rpk topic create 'glue-sample-my-event'
+
+system ok
+rpk topic produce -f '%v{hex}\n' 'glue-sample-my-event' <<EOF
+03005af405ef11b5444281a2e0563e5a734606666f6f80868dc8ebd98404
+EOF
+
+statement ok
+ALTER SYSTEM SET license_key TO '';
+
+statement error
+create source t with (
+  connector = 'kafka',
+  properties.bootstrap.server='${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}',
+  topic = 'glue-sample-my-event')
+format plain encode avro (
+  aws.glue.schema_arn = 'arn:aws:glue:ap-southeast-1:123456123456:schema/default-registry/MyEvent',
+  aws.glue.mock_config = '{
+    "by_id": {},
+    "arn_to_latest_id: {}"
+  }');
+----
+db error: ERROR: Failed to run the query
+
+Caused by these errors (recent errors listed first):
+  1: connector error
+  2: feature GlueSchemaRegistry is only available for tier Paid and above, while the current tier is Free
+
+Hint: You may want to set a license key with `ALTER SYSTEM SET license_key = '...';` command.
+
+
+statement ok
+ALTER SYSTEM SET license_key TO DEFAULT;
+
+statement ok
+create source t with (
+  connector = 'kafka',
+  properties.bootstrap.server='${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}',
+  topic = 'glue-sample-my-event')
+format plain encode avro (
+  aws.glue.schema_arn = 'arn:aws:glue:ap-southeast-1:123456123456:schema/default-registry/MyEvent',
+  aws.glue.mock_config = '{
+    "by_id":{
+      "5af405ef-11b5-4442-81a2-e0563e5a7346": {
+        "type": "record",
+        "name": "MyEvent",
+        "fields": [
+          {
+            "name": "f1",
+            "type": "string"
+          },
+          {
+            "name": "f2",
+            "type": {
+              "type": "long",
+              "logicalType": "timestamp-micros"
+            }
+          }
+        ]
+      }
+    },
+    "arn_to_latest_id":{
+      "arn:aws:glue:ap-southeast-1:123456123456:schema/default-registry/MyEvent": "5af405ef-11b5-4442-81a2-e0563e5a7346"
+    }
+  }');
+
+query TT
+select * from t;
+----
+foo 2006-01-02 22:04:05.123456+00:00
+
+statement ok
+alter source t format plain encode avro (
+  aws.glue.schema_arn = 'arn:aws:glue:ap-southeast-1:123456123456:schema/default-registry/MyEvent',
+  aws.glue.mock_config = '{
+    "by_id":{
+      "5af405ef-11b5-4442-81a2-e0563e5a7346": {
+        "type": "record",
+        "name": "MyEvent",
+        "fields": [
+          {
+            "name": "f1",
+            "type": "string"
+          },
+          {
+            "name": "f2",
+            "type": {
+              "type": "long",
+              "logicalType": "timestamp-micros"
+            }
+          }
+        ]
+      },
+      "4516411b-b1e7-4e67-839f-3ef1b8c29280": {
+        "type": "record",
+        "name": "MyEvent",
+        "fields": [
+          {
+            "name": "f1",
+            "type": "string"
+          },
+          {
+            "name": "f2",
+            "type": {
+              "type": "long",
+              "logicalType": "timestamp-micros"
+            }
+          },
+          {
+            "name": "f3",
+            "type": ["null", "bytes"],
+            "default": null
+          }
+        ]
+      }
+    },
+    "arn_to_latest_id":{
+      "arn:aws:glue:ap-southeast-1:123456123456:schema/default-registry/MyEvent": "4516411b-b1e7-4e67-839f-3ef1b8c29280"
+    }
+  }');
+
+query TTT
+select * from t;
+----
+foo 2006-01-02 22:04:05.123456+00:00 NULL
+
+system ok
+rpk topic produce -f '%v{hex}\n' 'glue-sample-my-event' <<EOF
+03004516411bb1e74e67839f3ef1b8c292800441428089b5e9a886ee050208deadbeef
+EOF
+
+query TTT
+select * from t order by 2;
+----
+foo 2006-01-02 22:04:05.123456+00:00 NULL
+AB  2022-04-08 00:00:00.123456+00:00 \xdeadbeef
+
+statement ok
+drop source t;
+
+system ok
+rpk topic delete 'glue-sample-my-event'
diff --git a/e2e_test/source_inline/kafka/avro/name_strategy.slt b/e2e_test/source_inline/kafka/avro/name_strategy.slt
index e48be738886f4..212c79e6f31a8 100644
--- a/e2e_test/source_inline/kafka/avro/name_strategy.slt
+++ b/e2e_test/source_inline/kafka/avro/name_strategy.slt
@@ -84,17 +84,6 @@ python3 scripts/source/schema_registry_producer.py  "${RISEDEV_KAFKA_BOOTSTRAP_S
 
 
 
-statement error SCHEMA_REGISTRY_NAME_STRATEGY_TOPIC_RECORD_NAME_STRATEGY expect non-empty field key\.message
-create table t_topic_record () with (
-    connector = 'kafka',
-    topic = 'upsert_avro_json-topic-record',
-    properties.bootstrap.server = '${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}'
-) format upsert encode avro (
-    schema.registry = '${RISEDEV_SCHEMA_REGISTRY_URL}',
-    schema.registry.name.strategy = 'topic_record_name_strategy',
-    message = 'CPLM.OBJ_ATTRIBUTE_VALUE'
-);
-
 statement ok
 create table t_topic_record (primary key(rw_key))
 INCLUDE KEY AS rw_key
@@ -105,8 +94,7 @@ with (
 ) format upsert encode avro (
     schema.registry = '${RISEDEV_SCHEMA_REGISTRY_URL}',
     schema.registry.name.strategy = 'topic_record_name_strategy',
-    message = 'CPLM.OBJ_ATTRIBUTE_VALUE',
-    key.message = 'string'
+    message = 'CPLM.OBJ_ATTRIBUTE_VALUE'
 );
 
 
diff --git a/e2e_test/source_inline/kafka/avro/union.slt b/e2e_test/source_inline/kafka/avro/union.slt
new file mode 100644
index 0000000000000..44e1db659d120
--- /dev/null
+++ b/e2e_test/source_inline/kafka/avro/union.slt
@@ -0,0 +1,175 @@
+control substitution on
+
+system ok
+rpk topic delete 'avro-union' || true; \
+(rpk sr subject delete 'avro-union-value' && rpk sr subject delete 'avro-union-value' --permanent) || true;
+rpk topic create avro-union
+
+system ok
+sr_register avro-union-value '
+{
+  "type": "record",
+  "name": "Root",
+  "fields": [
+    {
+      "name": "unionType",
+      "type": ["int", "string"]
+    },
+    {
+      "name": "unionTypeComplex",
+      "type": [
+        "null",
+        {"type": "record", "name": "Email","fields": [{"name":"inner","type":"string"}]},
+        {"type": "record", "name": "Fax","fields": [{"name":"inner","type":"int"}]},
+        {"type": "record", "name": "Sms","fields": [{"name":"inner","type":"int"}]}
+      ]
+    },
+    {
+        "name": "enumField",
+        "type": ["null", "int", {
+            "type": "enum",
+            "name": "myEnum",
+            "namespace": "my.namespace",
+            "symbols": ["A", "B", "C", "D"]
+        }],
+        "default": null
+    }
+  ]
+}
+'
+
+system ok
+cat<<EOF | rpk topic produce avro-union --schema-id=topic
+{"unionType": {"int":1}, "unionTypeComplex": {"Sms": {"inner":6}},  "enumField": {"my.namespace.myEnum": "A"}}
+{"unionType": {"string":"2"}, "unionTypeComplex": {"Fax": {"inner":6}}}
+{"unionType": {"int":3}, "unionTypeComplex": {"Email": {"inner":"a@b.c"}}, "enumField": {"int":66}}
+EOF
+
+statement error
+create source avro_union
+WITH (
+    ${RISEDEV_KAFKA_WITH_OPTIONS_COMMON},
+    topic = 'avro-union'
+)
+FORMAT PLAIN ENCODE AVRO (
+    schema.registry = '${RISEDEV_SCHEMA_REGISTRY_URL}'
+);
+----
+db error: ERROR: Failed to run the query
+
+Caused by these errors (recent errors listed first):
+  1: connector error
+  2: failed to convert Avro union to struct
+  3: Feature is not yet implemented: Avro named type used in Union type: Record(RecordSchema { name: Name { name: "Email", namespace: None }, aliases: None, doc: None, fields: [RecordField { name: "inner", doc: None, aliases: None, default: None, schema: String, order: Ascending, position: 0, custom_attributes: {} }], lookup: {"inner": 0}, attributes: {} })
+Tracking issue: https://github.com/risingwavelabs/risingwave/issues/17632
+
+
+# FIXME: The following is the current buggy result.
+
+
+# query ? rowsort
+# select * from avro_union
+# ----
+# ("(a@b.c)",,)
+# (,"(6)",)
+# (,"(6)",)
+
+# # Demonstrate how to access union variants (struct fields) below:
+# # Note that we need to use quotes.
+
+# query ? rowsort
+# select ("enumField")."my.namespace.myEnum" from avro_union;
+# ----
+# A
+# NULL
+# NULL
+
+# # To output the union’s tag (i.e. case in protobuf), a case-when can be used.
+# query ? rowsort
+# select
+# case
+#   when ("unionTypeComplex")."Sms" is not null then 'Sms'
+#   when ("unionTypeComplex")."Fax" is not null then 'Fax'
+#   when ("unionTypeComplex")."Email"  is not null then 'Email'
+#   else null -- optional
+# end
+# from avro_union;
+# ----
+# Email
+# Fax
+# Fax
+
+
+
+system ok
+rpk topic delete 'avro-union-simple' || true; \
+(rpk sr subject delete 'avro-union-simple-value' && rpk sr subject delete 'avro-union-simple-value' --permanent) || true;
+rpk topic create avro-union-simple
+
+system ok
+sr_register avro-union-simple-value '
+{
+  "type": "record",
+  "name": "Root",
+  "fields": [
+    {
+      "name": "unionType",
+      "type": ["int", "string", "null", "boolean"]
+    }
+  ]
+}
+'
+
+system ok
+cat<<EOF | rpk topic produce avro-union-simple --schema-id=topic
+{"unionType": {"int":1}}
+{"unionType": {"string":"2"}}
+{"unionType": {"boolean": true}}
+{"unionType": null}
+EOF
+
+statement ok
+create source avro_union
+WITH (
+    ${RISEDEV_KAFKA_WITH_OPTIONS_COMMON},
+    topic = 'avro-union-simple'
+)
+FORMAT PLAIN ENCODE AVRO (
+    schema.registry = '${RISEDEV_SCHEMA_REGISTRY_URL}'
+);
+
+
+query ? rowsort
+select * from avro_union
+----
+(,,t)
+(,2,)
+(1,,)
+NULL
+
+# Demonstrate how to access union variants (struct fields) below:
+# Note that we need to use quotes.
+
+query ? rowsort
+select ("unionType")."string" from avro_union;
+----
+2
+NULL
+NULL
+NULL
+
+# To output the union’s tag (i.e. case in protobuf), a case-when can be used.
+query ? rowsort
+select
+case
+  when ("unionType")."int" is not null then 'int'
+  when ("unionType")."string" is not null then 'string'
+  when ("unionType")."boolean"  is not null then 'boolean'
+  else null -- optional
+end
+from avro_union;
+----
+NULL
+boolean
+int
+string
diff --git a/e2e_test/source_inline/kafka/protobuf/basic.slt b/e2e_test/source_inline/kafka/protobuf/basic.slt
index 0eae891d04bcd..44153949e79e8 100644
--- a/e2e_test/source_inline/kafka/protobuf/basic.slt
+++ b/e2e_test/source_inline/kafka/protobuf/basic.slt
@@ -33,6 +33,22 @@ FORMAT plain ENCODE protobuf(
         message = 'test.User'
     );
 
+
+# for upsert protobuf source
+# NOTE: the key part is in json format and rw only read it as bytes
+statement ok
+create table sr_pb_upsert (primary key (rw_key))
+include
+    key as rw_key
+with (
+    ${RISEDEV_KAFKA_WITH_OPTIONS_COMMON},
+    topic = 'sr_pb_test',
+    scan.startup.mode = 'earliest')
+FORMAT plain ENCODE protobuf(
+        schema.registry = '${RISEDEV_SCHEMA_REGISTRY_URL}',
+        message = 'test.User'
+    );
+
 # Wait for source
 sleep 2s
 
@@ -50,9 +66,16 @@ select min(id), max(id), max((sc).file_name) from sr_pb_test;
 ----
 0 19 source/context_019.proto
 
+query TT
+select convert_from(min(rw_key), 'UTF-8'), convert_from(max(rw_key), 'UTF-8') from sr_pb_upsert;
+----
+{"id": 0} {"id": 9}
 
 statement ok
 drop table sr_pb_test;
 
 statement ok
 drop table sr_pb_test_bk;
+
+statement ok
+drop table sr_pb_upsert;
diff --git a/e2e_test/source_inline/kafka/protobuf/pb.py b/e2e_test/source_inline/kafka/protobuf/pb.py
index 4cab50f899e50..d78db1b536b9f 100644
--- a/e2e_test/source_inline/kafka/protobuf/pb.py
+++ b/e2e_test/source_inline/kafka/protobuf/pb.py
@@ -1,4 +1,5 @@
 import sys
+import json
 import importlib
 from google.protobuf.source_context_pb2 import SourceContext
 from confluent_kafka import Producer
@@ -55,6 +56,7 @@ def send_to_kafka(
         producer.produce(
             topic=topic,
             partition=0,
+            key=json.dumps({"id": i}), # RisingWave does not handle key schema, so we use json
             value=serializer(user, SerializationContext(topic, MessageField.VALUE)),
             on_delivery=delivery_report,
         )
diff --git a/e2e_test/source_inline/pubsub/prepare-data.rs b/e2e_test/source_inline/pubsub/prepare-data.rs
index a792c6cbb2618..e084b46919421 100755
--- a/e2e_test/source_inline/pubsub/prepare-data.rs
+++ b/e2e_test/source_inline/pubsub/prepare-data.rs
@@ -1,5 +1,5 @@
 #!/usr/bin/env -S cargo -Zscript
-```cargo
+---cargo
 [dependencies]
 anyhow = "1"
 google-cloud-googleapis = { version = "0.13", features = ["pubsub"] }
@@ -13,7 +13,7 @@ tokio = { version = "0.2", package = "madsim-tokio", features = [
     "signal",
     "fs",
 ] }
-```
+---
 
 use google_cloud_googleapis::pubsub::v1::PubsubMessage;
 use google_cloud_pubsub::client::{Client, ClientConfig};
diff --git a/e2e_test/streaming/aggregate/shuffle_approx_percentile.slt b/e2e_test/streaming/aggregate/shuffle_approx_percentile.slt
new file mode 100644
index 0000000000000..efc377f8aed48
--- /dev/null
+++ b/e2e_test/streaming/aggregate/shuffle_approx_percentile.slt
@@ -0,0 +1,104 @@
+# Single phase approx percentile
+statement ok
+create table t(p_col double, grp_col int);
+
+statement ok
+insert into t select a, 1 from generate_series(-1000, 1000) t(a);
+
+statement ok
+flush;
+
+query I
+ select
+     approx_percentile(0.01, 0.01) within group (order by p_col) as p01,
+     approx_percentile(0.1, 0.01) within group (order by p_col) as p10,
+     approx_percentile(0.5, 0.01) within group (order by p_col) as p50,
+     approx_percentile(0.9, 0.01) within group (order by p_col) as p90,
+     approx_percentile(0.99, 0.01) within group (order by p_col) as p99
+ from t group by grp_col;
+----
+-982.5779489474152 -804.4614206837127 0 804.4614206837127 982.5779489474152
+
+query I
+select
+  percentile_disc(0.01) within group (order by p_col) as p01,
+  percentile_disc(0.1) within group (order by p_col) as p10,
+  percentile_disc(0.5) within group (order by p_col) as p50,
+  percentile_disc(0.9) within group (order by p_col) as p90,
+  percentile_disc(0.99) within group (order by p_col) as p99
+from t group by grp_col;
+----
+-980 -800 0 800 980
+
+statement ok
+create materialized view m1 as
+ select
+     approx_percentile(0.01, 0.01) within group (order by p_col) as p01,
+     approx_percentile(0.1, 0.01) within group (order by p_col) as p10,
+     approx_percentile(0.5, 0.01) within group (order by p_col) as p50,
+     approx_percentile(0.9, 0.01) within group (order by p_col) as p90,
+     approx_percentile(0.99, 0.01) within group (order by p_col) as p99
+ from t group by grp_col;
+
+query I
+select * from m1;
+----
+-982.5779489474152 -804.4614206837127 0 804.4614206837127 982.5779489474152
+
+# Test state encode / decode
+onlyif can-use-recover
+statement ok
+recover;
+
+onlyif can-use-recover
+sleep 10s
+
+query I
+select * from m1;
+----
+-982.5779489474152 -804.4614206837127 0 804.4614206837127 982.5779489474152
+
+# Test state encode / decode
+onlyif can-use-recover
+statement ok
+recover;
+
+onlyif can-use-recover
+sleep 10s
+
+query I
+select * from m1;
+----
+-982.5779489474152 -804.4614206837127 0 804.4614206837127 982.5779489474152
+
+# Test 0<x<1 values
+statement ok
+insert into t select 0.001, 1 from generate_series(1, 500);
+
+statement ok
+insert into t select 0.0001, 1 from generate_series(1, 501);
+
+statement ok
+flush;
+
+query I
+select * from m1;
+----
+-963.1209598593477 -699.3618972397041 0.00009999833511933609 699.3618972397041 963.1209598593477
+
+query I
+select
+  percentile_disc(0.01) within group (order by p_col) as p01,
+  percentile_disc(0.1) within group (order by p_col) as p10,
+  percentile_disc(0.5) within group (order by p_col) as p50,
+  percentile_disc(0.9) within group (order by p_col) as p90,
+  percentile_disc(0.99) within group (order by p_col) as p99
+from t group by grp_col;
+----
+-970 -700 0.0001 700 970
+
+statement ok
+drop materialized view m1;
+
+statement ok
+drop table t;
\ No newline at end of file
diff --git a/e2e_test/streaming/aggregate/two_phase_approx_percentile.slt b/e2e_test/streaming/aggregate/two_phase_approx_percentile.slt
new file mode 100644
index 0000000000000..2f92d37f6e46b
--- /dev/null
+++ b/e2e_test/streaming/aggregate/two_phase_approx_percentile.slt
@@ -0,0 +1,78 @@
+# Single phase approx percentile
+statement ok
+create table t(p_col double, grp_col int);
+
+statement ok
+insert into t select a, 1 from generate_series(0, 10) t(a);
+
+statement ok
+insert into t values(0, 1);
+
+statement ok
+flush;
+
+statement ok
+create materialized view m1 as select
+     approx_percentile(0.01, 0.01) within group (order by p_col) as p01
+from t;
+
+statement ok
+flush;
+
+query I
+select * from m1;
+----
+0
+
+query I
+select percentile_cont(0.01) within group (order by p_col) from t;
+----
+0
+
+statement ok
+insert into t select a, 1 from generate_series(11, 1000) t(a);
+
+statement ok
+flush;
+
+query I
+select * from m1;
+----
+8.93541864376352
+
+query I
+select percentile_cont(0.01) within group (order by p_col) from t;
+----
+9.01
+
+query I
+select approx_percentile(0.01, 0.01) within group (order by p_col) from t group by grp_col;
+----
+8.93541864376352
+
+statement ok
+insert into t select a, 1 from generate_series(-1000, -1) t(a);
+
+statement ok
+flush;
+
+query I
+select * from m1;
+----
+-982.5779489474152
+
+query I
+select approx_percentile(0.01, 0.01) within group (order by p_col) from t group by grp_col;
+----
+-982.5779489474152
+
+query I
+select percentile_cont(0.01) within group (order by p_col) from t;
+----
+-979.99
+
+statement ok
+drop materialized view m1;
+
+statement ok
+drop table t;
\ No newline at end of file
diff --git a/e2e_test/streaming/aggregate/two_phase_approx_percentile_merge.slt b/e2e_test/streaming/aggregate/two_phase_approx_percentile_merge.slt
new file mode 100644
index 0000000000000..e168767b7b9be
--- /dev/null
+++ b/e2e_test/streaming/aggregate/two_phase_approx_percentile_merge.slt
@@ -0,0 +1,93 @@
+# Single phase approx percentile
+statement ok
+create table t(p_col double, grp_col int);
+
+statement ok
+insert into t select a, 1 from generate_series(-1000, 1000) t(a);
+
+statement ok
+flush;
+
+query I
+select
+  percentile_cont(0.01) within group (order by p_col) as p01,
+  percentile_cont(0.1) within group (order by p_col) as p10,
+  percentile_cont(0.5) within group (order by p_col) as p50,
+  percentile_cont(0.9) within group (order by p_col) as p90,
+  percentile_cont(0.99) within group (order by p_col) as p99
+from t;
+----
+-980 -800 0 800 980
+
+statement ok
+create materialized view m1 as
+ select
+     approx_percentile(0.01, 0.01) within group (order by p_col) as p01,
+     approx_percentile(0.1, 0.01) within group (order by p_col) as p10,
+     approx_percentile(0.5, 0.01) within group (order by p_col) as p50,
+     approx_percentile(0.9, 0.01) within group (order by p_col) as p90,
+     approx_percentile(0.99, 0.01) within group (order by p_col) as p99
+ from t;
+
+query I
+select * from m1;
+----
+-982.5779489474152 -804.4614206837127 0 804.4614206837127 982.5779489474152
+
+# Test state encode / decode
+onlyif can-use-recover
+statement ok
+recover;
+
+onlyif can-use-recover
+sleep 10s
+
+query I
+select * from m1;
+----
+-982.5779489474152 -804.4614206837127 0 804.4614206837127 982.5779489474152
+
+# Test state encode / decode
+onlyif can-use-recover
+statement ok
+recover;
+
+onlyif can-use-recover
+sleep 10s
+
+query I
+select * from m1;
+----
+-982.5779489474152 -804.4614206837127 0 804.4614206837127 982.5779489474152
+
+# Test 0<x<1 values
+statement ok
+insert into t select 0.001, 1 from generate_series(1, 500);
+
+statement ok
+insert into t select 0.0001, 1 from generate_series(1, 501);
+
+statement ok
+flush;
+
+query I
+select * from m1;
+----
+-963.1209598593477 -699.3618972397041 0.00009999833511933609 699.3618972397041 963.1209598593477
+
+query I
+select
+  percentile_cont(0.01) within group (order by p_col) as p01,
+  percentile_cont(0.1) within group (order by p_col) as p10,
+  percentile_cont(0.5) within group (order by p_col) as p50,
+  percentile_cont(0.9) within group (order by p_col) as p90,
+  percentile_cont(0.99) within group (order by p_col) as p99
+from t;
+----
+-969.99 -699.9 0.0001 699.9000000000001 969.9899999999998
+
+statement ok
+drop materialized view m1;
+
+statement ok
+drop table t;
\ No newline at end of file
diff --git a/e2e_test/streaming/rate_limit/alter_rate_limit.slt b/e2e_test/streaming/rate_limit/alter_rate_limit.slt
new file mode 100644
index 0000000000000..320662d2e5eea
--- /dev/null
+++ b/e2e_test/streaming/rate_limit/alter_rate_limit.slt
@@ -0,0 +1,50 @@
+onlyif can-use-recover
+statement ok
+CREATE TABLE t (v1 int);
+
+onlyif can-use-recover
+statement ok
+INSERT INTO t VALUES (1);
+
+onlyif can-use-recover
+statement ok
+flush;
+
+onlyif can-use-recover
+statement ok
+SET BACKGROUND_DDL=true;
+
+onlyif can-use-recover
+statement ok
+CREATE MATERIALIZED VIEW backfill_rate_limit_0 with ( backfill_rate_limit = 0 ) AS SELECT * FROM t;
+
+onlyif can-use-recover
+sleep 1s
+
+onlyif can-use-recover
+query I
+select btrim(progress, '%')::double from rw_ddl_progress limit 1;
+----
+0
+
+onlyif can-use-recover
+statement ok
+ALTER MATERIALIZED VIEW backfill_rate_limit_0 SET BACKFILL_RATE_LIMIT = 1;
+
+onlyif can-use-recover
+statement ok
+wait;
+
+onlyif can-use-recover
+query I
+select * from backfill_rate_limit_0;
+----
+1
+
+onlyif can-use-recover
+statement ok
+DROP MATERIALIZED VIEW backfill_rate_limit_0;
+
+onlyif can-use-recover
+statement ok
+DROP TABLE t;
\ No newline at end of file
diff --git a/e2e_test/streaming/rate_limit/alter_rate_limit.slt.ignore b/e2e_test/streaming/rate_limit/alter_rate_limit.slt.ignore
deleted file mode 100644
index 7afe279a80f1e..0000000000000
--- a/e2e_test/streaming/rate_limit/alter_rate_limit.slt.ignore
+++ /dev/null
@@ -1,35 +0,0 @@
--- This test is ignored until alter mv rate limit is fully supported.
-
-statement ok
-CREATE TABLE t (v1 int);
-
-statement ok
-INSERT INTO t VALUES (1);
-
-statement ok
-flush;
-
-statement ok
-SET BACKGROUND_DDL=true;
-
-statement ok
-CREATE MATERIALIZED VIEW streaming_rate_limit_0 with ( streaming_rate_limit = 0 ) AS SELECT * FROM t;
-
-skipif in-memory
-sleep 1s
-
-query I
-select progress from rw_ddl_progress;
-----
-0.00%
-
-statement ok
-ALTER MATERIALIZED VIEW streaming_rate_limit_0 SET STREAMING_RATE_LIMIT = 1;
-
-statement ok
-wait;
-
-query I
-select * from streaming_rate_limit_0;
-----
-1
\ No newline at end of file
diff --git a/e2e_test/streaming/rate_limit/basic.slt b/e2e_test/streaming/rate_limit/basic.slt
index 3d5464d604d8b..5fb3548197fae 100644
--- a/e2e_test/streaming/rate_limit/basic.slt
+++ b/e2e_test/streaming/rate_limit/basic.slt
@@ -5,7 +5,7 @@ statement ok
 CREATE TABLE t1(v1 int, v2 int);
 
 statement ok
-SET STREAMING_RATE_LIMIT TO 2000;
+SET BACKFILL_RATE_LIMIT TO 2000;
 
 statement ok
 CREATE MATERIALIZED VIEW m AS SELECT * FROM t1;
@@ -35,7 +35,7 @@ FLUSH;
 # Test with small rate_limit. In this case, the stream chunk needs to be split
 
 statement ok
-SET STREAMING_RATE_LIMIT TO 1;
+SET BACKFILL_RATE_LIMIT TO 1;
 
 statement ok
 CREATE MATERIALIZED VIEW m AS SELECT * FROM t1;
diff --git a/e2e_test/streaming/rate_limit/snapshot_amplification.slt b/e2e_test/streaming/rate_limit/snapshot_amplification.slt
index f704bc637c02c..087bc01a80465 100644
--- a/e2e_test/streaming/rate_limit/snapshot_amplification.slt
+++ b/e2e_test/streaming/rate_limit/snapshot_amplification.slt
@@ -6,7 +6,7 @@ statement ok
 SET STREAMING_PARALLELISM=2;
 
 statement ok
-SET STREAMING_RATE_LIMIT=1;
+SET BACKFILL_RATE_LIMIT=1;
 
 statement ok
 CREATE TABLE table (i1 int);
diff --git a/e2e_test/streaming/rate_limit/upstream_amplification.slt b/e2e_test/streaming/rate_limit/upstream_amplification.slt
index 63528472050a8..989a8310bf4ce 100644
--- a/e2e_test/streaming/rate_limit/upstream_amplification.slt
+++ b/e2e_test/streaming/rate_limit/upstream_amplification.slt
@@ -7,7 +7,7 @@ statement ok
 SET STREAMING_PARALLELISM=2;
 
 statement ok
-SET STREAMING_RATE_LIMIT=1;
+SET BACKFILL_RATE_LIMIT=1;
 
 statement ok
 CREATE TABLE source_table (i1 int)
diff --git a/e2e_test/streaming/union.slt b/e2e_test/streaming/union.slt
index 2e99097a84bc2..8bb5245bee722 100644
--- a/e2e_test/streaming/union.slt
+++ b/e2e_test/streaming/union.slt
@@ -2,10 +2,10 @@ statement ok
 SET RW_IMPLICIT_FLUSH TO true;
 
 statement ok
-create table t1 (v1 int, v2 int);
+create table t1 (v1 int, v2 int, v4 int);
 
 statement ok
-create table t2 (v1 int, v3 int);
+create table t2 (v1 int, v3 int, v4 int);
 
 statement ok
 create materialized view v as select * from t1 union all select * from t2;
@@ -13,6 +13,12 @@ create materialized view v as select * from t1 union all select * from t2;
 statement ok
 create materialized view v2 as select * from t1 union select * from t2;
 
+statement ok
+create materialized view v3 as select * from t1 union all corresponding select * from t2;
+
+statement ok
+create materialized view v4 as select * from t1 union corresponding by (v4, v1) select * from t2;
+
 query II
 select * from v;
 ----
@@ -22,57 +28,96 @@ select * from v2;
 ----
 
 statement ok
-insert into t1 values(1, 2);
+insert into t1 values(1, 2, 3);
 
-query II
+query III
 select * from v;
 ----
-1 2
+1 2 3
 
-query II
+query III
 select * from v2;
 ----
-1 2
+1 2 3
+
+query II
+select * from v3;
+----
+1 3
+
+query II
+select * from v4;
+----
+3 1
 
 statement ok
-insert into t2 values(1, 2);
+insert into t2 values(1, 2, 3);
 
 
-query II
+query III
 select * from v;
 ----
-1 2
-1 2
+1 2 3
+1 2 3
 
-query II
+query III
 select * from v2;
 ----
-1 2
+1 2 3
+
+query II
+select * from v3;
+----
+1 3
+1 3
+
+query II
+select * from v4;
+----
+3 1
 
 statement ok
 delete from t1 where v1 = 1;
 
-query II
+query III
 select * from v;
 ----
-1 2
+1 2 3
 
-query II
+query III
 select * from v2;
 ----
-1 2
+1 2 3
+
+query II
+select * from v3;
+----
+1 3
+
+query II
+select * from v4;
+----
+3 1
 
 statement ok
 delete from t2 where v1 = 1;
 
-query II
+query III
 select * from v;
 ----
 
-query II
+query III
 select * from v2;
 ----
 
+query II
+select * from v3;
+----
+
+query II
+select * from v4;
+----
+
 
 statement ok
 drop materialized view v;
@@ -80,6 +125,33 @@ drop materialized view v;
 statement ok
 drop materialized view v2;
 
+statement ok
+drop materialized view v3;
+
+statement ok
+drop materialized view v4;
+
+statement error
+create materialized view v5 as select * from t1 union corresponding by (vxx, v1) select * from t2
+----
+db error: ERROR: Failed to run the query
+
+Caused by:
+  Invalid input syntax: Column name `vxx` in CORRESPONDING BY is not found in a side of the UNION operation. It shall be included in both sides.
+
+
+statement ok
+create table txx (vxx int);
+
+statement error
+create materialized view v5 as select * from t1 union corresponding select * from txx
+----
+db error: ERROR: Failed to run the query
+
+Caused by:
+  Invalid input syntax: When CORRESPONDING is specified, at least one column of the left side shall have a column name that is the column name of some column of the right side in a UNION operation. Left side query column list: ("v1", "v2", "v4"). Right side query column list: ("vxx").
+
+
 statement ok
 drop table t1;
 
diff --git a/e2e_test/streaming/watermark.slt b/e2e_test/streaming/watermark.slt
index 5d8f189dfd965..a7bc67066fd1e 100644
--- a/e2e_test/streaming/watermark.slt
+++ b/e2e_test/streaming/watermark.slt
@@ -21,7 +21,7 @@ statement ok
 insert into t values ('2023-05-06 16:56:01', 1);
 
 skipif in-memory
-sleep 10s
+sleep 20s
 
 skipif in-memory
 query TI
diff --git a/e2e_test/subscription/create_table_and_subscription.slt b/e2e_test/subscription/create_table_and_subscription.slt
index fd43567bc52de..8ba73fae7ec91 100644
--- a/e2e_test/subscription/create_table_and_subscription.slt
+++ b/e2e_test/subscription/create_table_and_subscription.slt
@@ -8,4 +8,16 @@ statement ok
 flush;
 
 statement ok
-create subscription sub from t1 with(retention = '1D');
\ No newline at end of file
+create subscription sub from t1 with(retention = '1D');
+
+statement ok
+create table t2 (v1 int, v2 int);
+
+statement ok
+create table t3 (v1 int primary key, v2 int);
+
+statement ok
+create subscription sub2 from t3 with(retention = '1D');
+
+statement ok
+create sink s1 into t3 from t2;
\ No newline at end of file
diff --git a/e2e_test/subscription/drop_table_and_subscription.slt b/e2e_test/subscription/drop_table_and_subscription.slt
index 0df183a5b7793..d89401c9cddc8 100644
--- a/e2e_test/subscription/drop_table_and_subscription.slt
+++ b/e2e_test/subscription/drop_table_and_subscription.slt
@@ -2,4 +2,16 @@ statement ok
 drop subscription sub;
 
 statement ok
-drop table t1;
\ No newline at end of file
+drop table t1;
+
+statement ok
+drop sink s1;
+
+statement ok
+drop subscription sub2;
+
+statement ok
+drop table t3;
+
+statement ok
+drop table t2;
\ No newline at end of file
diff --git a/e2e_test/subscription/main.py b/e2e_test/subscription/main.py
index 3ffaefd02cee6..caa1d3a141c09 100644
--- a/e2e_test/subscription/main.py
+++ b/e2e_test/subscription/main.py
@@ -262,7 +262,7 @@ def test_cursor_with_table_alter():
     drop_table_subscription()
 
 def test_cursor_fetch_n():
-    print(f"test_cursor_with_table_alter")
+    print(f"test_cursor_fetch_n")
     create_table_subscription()
     conn = psycopg2.connect(
         host="localhost",
@@ -304,6 +304,27 @@ def test_cursor_fetch_n():
     check_rows_data([10,100],row[3],3)
     drop_table_subscription()
 
+def test_rebuild_table():
+    print(f"test_rebuild_table")
+    create_table_subscription()
+    conn = psycopg2.connect(
+        host="localhost",
+        port="4566",
+        user="root",
+        database="dev"
+    )
+
+    execute_insert("declare cur subscription cursor for sub2",conn)
+    execute_insert("insert into t2 values(1,1)",conn)
+    execute_insert("flush",conn)
+    execute_insert("update t2 set v2 = 100 where v1 = 1",conn)
+    execute_insert("flush",conn)
+    row = execute_query("fetch 4 from cur",conn)
+    assert len(row) == 3
+    check_rows_data([1,1],row[0],1)
+    check_rows_data([1,1],row[1],4)
+    check_rows_data([1,100],row[2],3)
+
 if __name__ == "__main__":
     test_cursor_snapshot()
     test_cursor_op()
@@ -313,3 +334,4 @@ def test_cursor_fetch_n():
     test_cursor_since_begin()
     test_cursor_with_table_alter()
     test_cursor_fetch_n()
+    test_rebuild_table()
diff --git a/e2e_test/time_travel/basic.slt b/e2e_test/time_travel/basic.slt
new file mode 100644
index 0000000000000..962fd8e096a59
--- /dev/null
+++ b/e2e_test/time_travel/basic.slt
@@ -0,0 +1,38 @@
+statement ok
+SET RW_IMPLICIT_FLUSH TO true;
+
+statement ok
+CREATE TABLE t (k INT);
+
+query I
+SELECT *  FROM t;
+----
+
+sleep 5s
+
+statement ok
+INSERT INTO t VALUES (1);
+
+query I
+SELECT *  FROM t;
+----
+1
+
+query I
+SELECT *  FROM t FOR SYSTEM_TIME AS OF now();
+----
+1
+
+query I
+SELECT *  FROM t FOR SYSTEM_TIME AS OF now() - '5' second;
+----
+
+sleep 5s
+
+query I
+SELECT *  FROM t FOR SYSTEM_TIME AS OF now() - '5' second;
+----
+1
+
+statement ok
+DROP TABLE t;
\ No newline at end of file
diff --git a/e2e_test/time_travel/index.slt b/e2e_test/time_travel/index.slt
new file mode 100644
index 0000000000000..4c81bc6838379
--- /dev/null
+++ b/e2e_test/time_travel/index.slt
@@ -0,0 +1,40 @@
+statement ok
+SET RW_IMPLICIT_FLUSH TO true;
+
+statement ok
+CREATE TABLE t (k INT);
+
+statement ok
+CREATE INDEX idx_t_k on t (k);
+
+sleep 5s
+
+statement ok
+INSERT INTO t VALUES (1);
+
+query I
+SELECT *  FROM t WHERE k=1;
+----
+1
+
+query I
+SELECT *  FROM t FOR SYSTEM_TIME AS OF now()  WHERE k=1;
+----
+1
+
+query I
+SELECT *  FROM t FOR SYSTEM_TIME AS OF now() - '5' second WHERE k=1;
+----
+
+sleep 5s
+
+query I
+SELECT *  FROM t FOR SYSTEM_TIME AS OF now() - '5' second WHERE k=1;
+----
+1
+
+statement ok
+DROP INDEX idx_t_k;
+
+statement ok
+DROP TABLE t;
\ No newline at end of file
diff --git a/e2e_test/time_travel/join.slt b/e2e_test/time_travel/join.slt
new file mode 100644
index 0000000000000..cfada20463f95
--- /dev/null
+++ b/e2e_test/time_travel/join.slt
@@ -0,0 +1,54 @@
+statement ok
+SET RW_IMPLICIT_FLUSH TO true;
+
+statement ok
+CREATE TABLE t1 (k1 INT);
+
+statement ok
+CREATE TABLE t2 (k2 INT);
+
+sleep 5s
+
+statement ok
+INSERT INTO t1 VALUES (1);
+
+statement ok
+INSERT INTO t2 VALUES (1);
+
+query I
+SELECT count(*) FROM t1 join t2 on t1.k1=t2.k2;
+----
+1
+
+query I
+SELECT count(*) FROM t1 FOR SYSTEM_TIME AS OF now() join t2  FOR SYSTEM_TIME AS OF now() on t1.k1=t2.k2;
+----
+1
+
+query I
+SELECT count(*) FROM t1 FOR SYSTEM_TIME AS OF now() - '5' second join t2  FOR SYSTEM_TIME AS OF now() on t1.k1=t2.k2;
+----
+0
+
+query I
+SELECT count(*) FROM t1 FOR SYSTEM_TIME AS OF now() join t2  FOR SYSTEM_TIME AS OF now() - '5' second on t1.k1=t2.k2;
+----
+0
+
+sleep 5s
+
+query I
+SELECT count(*) FROM t1 FOR SYSTEM_TIME AS OF now() - '5' second join t2  FOR SYSTEM_TIME AS OF now() on t1.k1=t2.k2;
+----
+1
+
+query I
+SELECT count(*) FROM t1 FOR SYSTEM_TIME AS OF now() join t2  FOR SYSTEM_TIME AS OF now() - '5' second on t1.k1=t2.k2;
+----
+1
+
+statement ok
+DROP TABLE t2;
+
+statement ok
+DROP TABLE t1;
\ No newline at end of file
diff --git a/e2e_test/time_travel/license.slt b/e2e_test/time_travel/license.slt
new file mode 100644
index 0000000000000..c5d200c051166
--- /dev/null
+++ b/e2e_test/time_travel/license.slt
@@ -0,0 +1,37 @@
+statement ok
+SET RW_IMPLICIT_FLUSH TO true;
+
+statement ok
+SET QUERY_MODE TO local;
+
+statement ok
+ALTER SYSTEM SET license_key TO '';
+
+statement ok
+CREATE TABLE t (k INT);
+
+query error
+SELECT *  FROM t FOR SYSTEM_TIME AS OF now();
+----
+db error: ERROR: Failed to run the query
+
+Caused by these errors (recent errors listed first):
+  1: Scheduler error
+  2: feature TimeTravel is only available for tier Paid and above, while the current tier is Free
+
+Hint: You may want to set a license key with `ALTER SYSTEM SET license_key = '...';` command.
+
+
+
+statement ok
+ALTER SYSTEM SET license_key TO DEFAULT;
+
+query I
+SELECT * FROM t FOR SYSTEM_TIME  AS OF now();
+----
+
+statement ok
+DROP TABLE t;
+
+statement ok
+SET QUERY_MODE TO auto;
\ No newline at end of file
diff --git a/e2e_test/time_travel/lookup_join.slt b/e2e_test/time_travel/lookup_join.slt
new file mode 100644
index 0000000000000..272bc7a34ae47
--- /dev/null
+++ b/e2e_test/time_travel/lookup_join.slt
@@ -0,0 +1,54 @@
+statement ok
+SET RW_IMPLICIT_FLUSH TO true;
+
+statement ok
+CREATE TABLE t1 (k1 INT);
+
+statement ok
+CREATE TABLE t2 (k2 INT PRIMARY KEY);
+
+sleep 5s
+
+statement ok
+INSERT INTO t1 VALUES (1);
+
+statement ok
+INSERT INTO t2 VALUES (1);
+
+query I
+SELECT count(*) FROM t1 join t2 on t1.k1=t2.k2;
+----
+1
+
+query I
+SELECT count(*) FROM t1 FOR SYSTEM_TIME AS OF now() join t2  FOR SYSTEM_TIME AS OF now() on t1.k1=t2.k2;
+----
+1
+
+query I
+SELECT count(*) FROM t1 FOR SYSTEM_TIME AS OF now() - '5' second join t2  FOR SYSTEM_TIME AS OF now() on t1.k1=t2.k2;
+----
+0
+
+query I
+SELECT count(*) FROM t1 FOR SYSTEM_TIME AS OF now() join t2  FOR SYSTEM_TIME AS OF now() - '5' second on t1.k1=t2.k2;
+----
+0
+
+sleep 5s
+
+query I
+SELECT count(*) FROM t1 FOR SYSTEM_TIME AS OF now() - '5' second join t2  FOR SYSTEM_TIME AS OF now() on t1.k1=t2.k2;
+----
+1
+
+query I
+SELECT count(*) FROM t1 FOR SYSTEM_TIME AS OF now() join t2  FOR SYSTEM_TIME AS OF now() - '5' second on t1.k1=t2.k2;
+----
+1
+
+statement ok
+DROP TABLE t2;
+
+statement ok
+DROP TABLE t1;
\ No newline at end of file
diff --git a/e2e_test/time_travel/syntax.slt b/e2e_test/time_travel/syntax.slt
new file mode 100644
index 0000000000000..6c3408a276763
--- /dev/null
+++ b/e2e_test/time_travel/syntax.slt
@@ -0,0 +1,58 @@
+statement ok
+SET RW_IMPLICIT_FLUSH TO true;
+
+statement ok
+SET QUERY_MODE TO local;
+
+statement ok
+CREATE TABLE t (k INT);
+
+query error
+SELECT * FROM t FOR SYSTEM_TIME AS OF 963716300;
+----
+db error: ERROR: Failed to run the query
+
+Caused by these errors (recent errors listed first):
+  1: gRPC request to batch service failed: Internal error
+  2: Storage error
+  3: Hummock error
+  4: Meta error: gRPC request to hummock service failed: Internal error: time travel: version not found for epoch 0
+
+
+query error
+SELECT * FROM t FOR SYSTEM_TIME AS OF '2000-02-20T12:13:14-08:30';
+----
+db error: ERROR: Failed to run the query
+
+Caused by these errors (recent errors listed first):
+  1: gRPC request to batch service failed: Internal error
+  2: Storage error
+  3: Hummock error
+  4: Meta error: gRPC request to hummock service failed: Internal error: time travel: version not found for epoch 0
+
+
+query error
+SELECT * FROM t FOR SYSTEM_TIME AS OF NOW() - '100' YEAR;
+----
+db error: ERROR: Failed to run the query
+
+Caused by these errors (recent errors listed first):
+  1: gRPC request to batch service failed: Internal error
+  2: Storage error
+  3: Hummock error
+  4: Meta error: gRPC request to hummock service failed: Internal error: time travel: version not found for epoch 0
+
+
+query I
+SELECT *  FROM t FOR SYSTEM_TIME AS OF '2130-02-10T12:13:14-08:30';
+----
+
+query I
+SELECT *  FROM t FOR SYSTEM_TIME AS OF 5066075780;
+----
+
+statement ok
+DROP TABLE t;
+
+statement ok
+SET QUERY_MODE TO auto;
\ No newline at end of file
diff --git a/e2e_test/udf/deno_udf.slt b/e2e_test/udf/deno_udf.slt
deleted file mode 100644
index 35e16daea16a2..0000000000000
--- a/e2e_test/udf/deno_udf.slt
+++ /dev/null
@@ -1,233 +0,0 @@
-statement ok
-CREATE FUNCTION gcd(a int, b int) RETURNS int LANGUAGE javascript RUNTIME deno AS $$
-    if(a == null || b == null) {
-        return null;
-    }
-    while (b != 0) {
-        let t = b;
-        b = a % b;
-        a = t;
-    }
-    return a;
-$$;
-
-query I
-select gcd(25, 15);
-----
-5
-
-statement ok
-drop function gcd;
-
-statement ok
-create function decimal_add(a decimal, b decimal) returns decimal language javascript RUNTIME deno as $$
-    return a.add(b);
-$$;
-
-query R
-select decimal_add(1.11, 2.22);
-----
-3.33
-
-statement ok
-drop function decimal_add;
-
-
-statement ok
-create function to_string(a boolean, b smallint, c int, d bigint, e real, f float, g decimal, h varchar, i bytea, j jsonb) returns varchar language javascript RUNTIME deno as $$
-    return a.toString() + b.toString() + c.toString() + d.toString() + e.toString() + f.toString() + g.toString() + h.toString() + i.toString() + JSON.stringify(j);
-$$;
-
-query T
-select to_string(false, 1::smallint, 2, 3, 4.5, 6.7, 8.9, 'abc', '\x010203', '{"key": 1}');
-----
-false1234.56.78.9abc1,2,3{"key":1}
-
-statement ok
-drop function to_string;
-
-# show data types in javascript
-statement ok
-create function js_typeof(a boolean, b smallint, c int, d bigint, e real, f float, g decimal, h varchar, i bytea, j jsonb) returns jsonb language javascript RUNTIME deno as $$
-    return {
-        boolean: typeof a,
-        smallint: typeof b,
-        int: typeof c,
-        bigint: typeof d,
-        real: typeof e,
-        float: typeof f,
-        decimal: typeof g,
-        varchar: typeof h,
-        bytea: typeof i,
-        jsonb: typeof j,
-    };
-$$;
-
-query T
-select js_typeof(false, 1::smallint, 2, 3, 4.5, 6.7, 8.9, 'abc', '\x010203', '{"key": 1}');
-----
-{"bigint": "bigint", "boolean": "boolean", "bytea": "object", "decimal": "object", "float": "number", "int": "number", "jsonb": "object", "real": "number", "smallint": "number", "varchar": "string"}
-
-statement ok
-drop function js_typeof;
-
-statement ok
-create function return_all(a boolean, b smallint, c int, d bigint, e real, f float, g decimal, h varchar, i bytea, j jsonb, s struct<f1 int, f2 int>)
-returns struct<a boolean, b smallint, c int, d bigint, e real, f float, g decimal, h varchar, i bytea, j jsonb, s struct<f1 int, f2 int>>
-language javascript runtime deno as $$
-    return {a,b,c,d,e,f,g,h,i,j,s};
-$$;
-
-query T
-select (return_all(
-    true,
-    1 ::smallint,
-    1,
-    1,
-    1,
-    1,
-    12345678901234567890.12345678,
-    'string',
-    'bytes',
-    '{"key":1}',
-    row(1, 2)::struct<f1 int, f2 int>
-)).*;
-----
-t 1 1 1 1 1 12345678901234567890.12345678 string \x6279746573 {"key": 1} (1,2)
-
-statement ok
-drop function return_all;
-
-
-statement ok
-create function series(n int) returns table (x int) language javascript RUNTIME deno as $$
-    for(let i = 0; i < n; i++) {
-        yield i;
-    }
-$$;
-
-query I
-select series(5);
-----
-0
-1
-2
-3
-4
-
-statement ok
-drop function series;
-
-
-statement ok
-create function split(s varchar) returns table (word varchar, length int) language javascript RUNTIME deno as $$
-    for(let word of s.split(' ')) {
-        yield { word: word, length: word.length };
-    }
-$$;
-
-query IT
-select * from split('rising wave');
-----
-rising 6
-wave   4
-
-statement ok
-drop function split;
-
-
-statement ok
-CREATE FUNCTION digest( t string ) RETURNS bytea LANGUAGE javascript RUNTIME deno AS $$
-    const subtle = crypto.subtle;
-    const key = await subtle.generateKey({
-        name: 'HMAC',
-        hash: 'SHA-256',
-        length: 256,
-        }, true, ['sign', 'verify']);
-        const enc = new TextEncoder();
-        const message = enc.encode(t);
-        const result = await subtle.sign({
-        name: 'HMAC',
-        }, key, message);
-    return result;
-$$ ASYNC;
-
-query I
-select bit_length(digest('Hello'));
-----
-256
-
-statement ok
-drop function digest;
-
-statement ok
-CREATE FUNCTION delay_response()
-    RETURNS TABLE (x int) LANGUAGE javascript RUNTIME deno AS $$
-        const delayedResponses = {
-            delays: [50, 10, 15],
-            wait(delay) {
-                return new Promise((resolve) => {
-                setTimeout(resolve, delay);
-                });
-            },
-            async *[Symbol.asyncIterator]() {
-                for (const delay of this.delays) {
-                await this.wait(delay);
-                yield delay;
-                }
-            },
-        };
-        return delayedResponses;
-$$ SYNC;
-
-query I
-select * FROM delay_response();
-----
-50
-10
-15
-
-statement ok
-drop function delay_response;
-
-system ok
-python3 e2e_test/udf/mock_server.py &
-
-# wait for server to start
-sleep 1s
-
-statement ok
-CREATE FUNCTION call_sse() RETURNS TABLE ( data struct<data struct<greetings string>>) LANGUAGE javascript RUNTIME deno USING LINK 'fs://e2e_test/udf/sse/bundled.table.js' SYNC;
-
-query I
-select * FROM call_sse();
-----
-(Hi)
-(Bonjour)
-(Hola)
-(Ciao)
-(Zdravo)
-
-statement ok
-drop function call_sse;
-
-statement ok
-CREATE FUNCTION fetch_api() RETURNS TABLE ( data struct< idx int>) LANGUAGE javascript RUNTIME deno AS $$
-    const response = await fetch('http://127.0.0.1:4200');
-    const resp = await response.json();
-    for (const r of resp.results) {
-        yield r;
-    }
-$$ ASYNC GENERATOR;
-
-query I
-select * FROM fetch_api();
-----
-1
-2
-
-statement ok
-drop function fetch_api;
-
-system ok
-pkill -9 python3
diff --git a/e2e_test/udf/python_udf.slt b/e2e_test/udf/python_udf.slt
index fedb9644160a5..8020e3cf9e7fc 100644
--- a/e2e_test/udf/python_udf.slt
+++ b/e2e_test/udf/python_udf.slt
@@ -251,6 +251,32 @@ statement ok
 drop aggregate weighted_avg;
 
 
+# UDF as aggregate function
+statement ok
+create function median(int[]) returns float language python as $$
+def median(values):
+    values.sort()
+    n = len(values)
+    if n % 2 == 0:
+        return (values[n // 2 - 1] + values[n // 2]) / 2
+    else:
+        return values[n // 2]
+$$;
+
+query F
+select aggregate:median(x) from (values (1), (2), (3), (4), (5)) as t(x);
+----
+3
+
+query F
+select aggregate:median(x) from (values (4), (3), (2), (1)) as t(x);
+----
+2.5
+
+statement ok
+drop function median;
+
+
 statement ok
 create function mismatched_arguments() returns int language python as $$
 def mismatched_arguments(x):
diff --git a/grafana/risingwave-dev-dashboard.dashboard.py b/grafana/risingwave-dev-dashboard.dashboard.py
index 1d0bab6a73d41..4c11b3cab52a9 100644
--- a/grafana/risingwave-dev-dashboard.dashboard.py
+++ b/grafana/risingwave-dev-dashboard.dashboard.py
@@ -460,44 +460,59 @@ def section_compaction(outer_panels):
                     ],
                 ),
                 panels.timeseries_bytes(
-                    "Hummock Sstable Size",
-                    "Total bytes gotten from sstable_bloom_filter, for observing bloom_filter size",
+                    "Hummock Sstable Bloom Filter Size",
+                    "For observing bloom_filter size, sstable file size, sstable block size etc.",
                     [
                         panels.target(
-                            f"sum by(le, {COMPONENT_LABEL}, {NODE_LABEL})(rate({metric('compactor_sstable_bloom_filter_size_sum')}[$__rate_interval]))  / sum by(le, {COMPONENT_LABEL}, {NODE_LABEL})(rate({metric('compactor_sstable_bloom_filter_size_count')}[$__rate_interval])) > 0",
-                            "avg_meta - {{%s}} @ {{%s}}"
-                            % (COMPONENT_LABEL, NODE_LABEL),
+                             f"histogram_quantile(0.50, sum(rate({metric('compactor_sstable_bloom_filter_size_bucket')}[$__rate_interval])) by (le, {COMPONENT_LABEL}, {NODE_LABEL}))",
+                            "bloom_filter_size_p50 - {{%s}} @ {{%s}}" % (COMPONENT_LABEL, NODE_LABEL),
                         ),
                         panels.target(
-                            f"sum by(le, {COMPONENT_LABEL}, {NODE_LABEL})(rate({metric('compactor_sstable_file_size_sum')}[$__rate_interval]))  / sum by(le, {COMPONENT_LABEL}, {NODE_LABEL})(rate({metric('compactor_sstable_file_size_count')}[$__rate_interval])) > 0",
-                            "avg_file - {{%s}} @ {{%s}}"
-                            % (COMPONENT_LABEL, NODE_LABEL),
+                             f"histogram_quantile(0.90, sum(rate({metric('compactor_sstable_bloom_filter_size_bucket')}[$__rate_interval])) by (le, {COMPONENT_LABEL}, {NODE_LABEL}))",
+                            "bloom_filter_size_p90 - {{%s}} @ {{%s}}" % (COMPONENT_LABEL, NODE_LABEL),
                         ),
                     ],
                 ),
                 panels.timeseries_bytes(
-                    "Hummock Sstable Item Size",
-                    "Total bytes gotten from sstable_avg_key_size, for observing sstable_avg_key_size",
+                    "Hummock Sstable File Size",
+                    "For observing sstable file size",
                     [
                         panels.target(
-                            f"sum by(le, {COMPONENT_LABEL}, {NODE_LABEL})(rate({metric('compactor_sstable_avg_key_size_sum')}[$__rate_interval]))  / sum by(le, {COMPONENT_LABEL}, {NODE_LABEL})(rate({metric('compactor_sstable_avg_key_size_count')}[$__rate_interval])) > 0",
-                            "avg_key_size - {{%s}} @ {{%s}}"
-                            % (COMPONENT_LABEL, NODE_LABEL),
+                             f"histogram_quantile(0.50, sum(rate({metric('compactor_sstable_file_size_bucket')}[$__rate_interval])) by (le, {COMPONENT_LABEL}, {NODE_LABEL}))",
+                            "sstable_file_size_p50 - {{%s}} @ {{%s}}" % (COMPONENT_LABEL, NODE_LABEL),
                         ),
                         panels.target(
-                            f"sum by(le, {COMPONENT_LABEL}, {NODE_LABEL})(rate({metric('compactor_sstable_avg_value_size_sum')}[$__rate_interval]))  / sum by(le, {COMPONENT_LABEL}, {NODE_LABEL})(rate({metric('compactor_sstable_avg_value_size_count')}[$__rate_interval]))",
-                            "avg_value_size - {{%s}} @ {{%s}}"
-                            % (COMPONENT_LABEL, NODE_LABEL),
+                             f"histogram_quantile(0.90, sum(rate({metric('compactor_sstable_file_size_bucket')}[$__rate_interval])) by (le, {COMPONENT_LABEL}, {NODE_LABEL}))",
+                            "sstable_file_size_p90 - {{%s}} @ {{%s}}" % (COMPONENT_LABEL, NODE_LABEL),
                         ),
                     ],
                 ),
-                panels.timeseries_count(
-                    "Hummock Sstable Stat",
-                    "Avg count gotten from sstable_distinct_epoch_count, for observing sstable_distinct_epoch_count",
+                panels.timeseries_bytes(
+                    "Hummock Sstable Block Size",
+                    "For observing sstable block size",
+                    [
+                        panels.target(
+                             f"histogram_quantile(0.50, sum(rate({metric('compactor_sstable_block_size_bucket')}[$__rate_interval])) by (le, {COMPONENT_LABEL}, {NODE_LABEL}))",
+                            "sstable_block_size_p50 - {{%s}} @ {{%s}}" % (COMPONENT_LABEL, NODE_LABEL),
+                        ),
+                        panels.target(
+                             f"histogram_quantile(0.90, sum(rate({metric('compactor_sstable_block_size_bucket')}[$__rate_interval])) by (le, {COMPONENT_LABEL}, {NODE_LABEL}))",
+                            "sstable_block_size_p90 - {{%s}} @ {{%s}}" % (COMPONENT_LABEL, NODE_LABEL),
+                        ),
+                    ],
+                ),
+                panels.timeseries_bytes(
+                    "Hummock Sstable Avg Key And Value Count",
+                    "For observing avg key and value count",
                     [
                         panels.target(
-                            f"sum by(le, {COMPONENT_LABEL}, {NODE_LABEL})(rate({metric('compactor_sstable_distinct_epoch_count_sum')}[$__rate_interval]))  / sum by(le, {COMPONENT_LABEL}, {NODE_LABEL})(rate({metric('compactor_sstable_distinct_epoch_count_count')}[$__rate_interval])) > 0",
-                            "avg_epoch_count - {{%s}} @ {{%s}}"
+                            f"sum by(le, {COMPONENT_LABEL}, {NODE_LABEL})(rate({metric('compactor_sstable_avg_key_size_sum')}[$__rate_interval]))  / sum by(le, {COMPONENT_LABEL}, {NODE_LABEL})(rate({metric('compactor_sstable_avg_key_size_count')}[$__rate_interval])) > 0",
+                            "avg_key_size - {{%s}} @ {{%s}}"
+                            % (COMPONENT_LABEL, NODE_LABEL),
+                        ),
+                        panels.target(
+                            f"sum by(le, {COMPONENT_LABEL}, {NODE_LABEL})(rate({metric('compactor_sstable_avg_value_size_sum')}[$__rate_interval]))  / sum by(le, {COMPONENT_LABEL}, {NODE_LABEL})(rate({metric('compactor_sstable_avg_value_size_count')}[$__rate_interval]))",
+                            "avg_value_size - {{%s}} @ {{%s}}"
                             % (COMPONENT_LABEL, NODE_LABEL),
                         ),
                     ],
@@ -826,9 +841,11 @@ def section_streaming(outer_panels):
                         ),
                     ],
                 ),
+                # TODO: These 2 metrics should be deprecated because they are unaware of Log Store
+                # Let's remove them when all sinks are migrated to Log Store
                 panels.timeseries_rowsps(
-                    "Sink Throughput(rows/s)",
-                    "The number of rows streamed into each sink per second.",
+                    "Sink Throughput(rows/s) *",
+                    "The number of rows streamed into each sink per second. For sinks with 'sink_decouple = true', please refer to the 'Sink Metrics' section",
                     [
                         panels.target(
                             f"sum(rate({metric('stream_sink_input_row_count')}[$__rate_interval])) by (sink_id) * on(sink_id) group_left(sink_name) group({metric('sink_info')}) by (sink_id, sink_name)",
@@ -837,8 +854,8 @@ def section_streaming(outer_panels):
                     ],
                 ),
                 panels.timeseries_rowsps(
-                    "Sink Throughput(rows/s) per Partition",
-                    "The number of rows streamed into each sink per second.",
+                    "Sink Throughput(rows/s) per Partition *",
+                    "The number of rows streamed into each sink per second. For sinks with 'sink_decouple = true', please refer to the 'Sink Metrics' section",
                     [
                         panels.target(
                             f"sum(rate({metric('stream_sink_input_row_count')}[$__rate_interval])) by (sink_id, actor_id) * on(actor_id) group_left(sink_name) {metric('sink_info')}",
@@ -1232,21 +1249,17 @@ def section_streaming_actors(outer_panels):
                         ),
                     ],
                 ),
-                panels.timeseries_actor_latency(
-                    "Executor Barrier Align",
+                panels.timeseries_percentage(
+                    "Executor Barrier Align Per Second",
                     "",
                     [
-                        *quantile(
-                            lambda quantile, legend: panels.target(
-                                f"histogram_quantile({quantile}, sum(rate({metric('stream_barrier_align_duration_bucket')}[$__rate_interval])) by (le, executor, fragment_id, wait_side, {COMPONENT_LABEL}))",
-                                f"p{legend} - executor {{{{executor}}}} fragment {{{{fragment_id}}}} {{{{wait_side}}}} - {{{{{COMPONENT_LABEL}}}}}",
-                            ),
-                            [90, 99, 999, "max"],
-                        ),
                         panels.target(
-                            f"sum by(le, executor, fragment_id, wait_side, job)(rate({metric('stream_barrier_align_duration_sum')}[$__rate_interval])) / sum by(le,executor,fragment_id,wait_side,{COMPONENT_LABEL}) (rate({metric('stream_barrier_align_duration_count')}[$__rate_interval])) > 0",
-                            "avg - executor {{executor}} fragment {{fragment_id}} {{wait_side}} - {{%s}}"
-                            % COMPONENT_LABEL,
+                            f"avg(rate({metric('stream_barrier_align_duration_ns')}[$__rate_interval]) / 1000000000) by (fragment_id,wait_side, executor)",
+                            "fragment {{fragment_id}} {{wait_side}} {{executor}}",
+                        ),
+                        panels.target_hidden(
+                            f"rate({metric('stream_barrier_align_duration_ns')}[$__rate_interval]) / 1000000000",
+                            "actor {{actor_id}} fragment {{fragment_id}} {{wait_side}} {{executor}}",
                         ),
                     ],
                 ),
@@ -3798,7 +3811,7 @@ def section_iceberg_metrics(outer_panels):
                     [
                         panels.target(
                             f"{metric('iceberg_write_qps')}",
-                            "{{executor_id}} @ {{sink_id}}",
+                            "{{sink_id}} {{sink_name}} actor {{actor_id}}",
                         ),
                     ],
                 ),
@@ -3808,14 +3821,14 @@ def section_iceberg_metrics(outer_panels):
                     [
                         *quantile(
                             lambda quantile, legend: panels.target(
-                                f"histogram_quantile({quantile}, sum(rate({metric('iceberg_write_latency_bucket')}[$__rate_interval])) by (le, sink_id))",
-                                f"p{legend}" + " @ {{sink_id}}",
+                                f"histogram_quantile({quantile}, sum(rate({metric('iceberg_write_latency_bucket')}[$__rate_interval])) by (le, sink_id, sink_name))",
+                                f"p{legend}" + " @ {{sink_id}} {{sink_name}}",
                             ),
                             [50, 99, "max"],
                         ),
                         panels.target(
-                            f"sum by(le, sink_id)(rate({metric('iceberg_write_latency_sum')}[$__rate_interval])) / sum by(le, type, job, instance) (rate({metric('iceberg_write_latency_count')}[$__rate_interval])) > 0",
-                            "avg @ {{sink_id}}",
+                            f"sum by(le, type, job, instance, sink_id, sink_name)(rate({metric('iceberg_write_latency_sum')}[$__rate_interval])) / sum by(le, type, job, instance, sink_id, sink_name) (rate({metric('iceberg_write_latency_count')}[$__rate_interval])) > 0",
+                            "avg @ {{sink_id}} {{sink_name}}",
                         ),
                     ],
                 ),
@@ -3824,8 +3837,8 @@ def section_iceberg_metrics(outer_panels):
                     "",
                     [
                         panels.target(
-                            f"{metric('iceberg_rolling_unfushed_data_file')}",
-                            "{{executor_id}} @ {{sink_id}}",
+                            f"{metric('iceberg_rolling_unflushed_data_file')}",
+                            "{{sink_id}} {{sink_name}} actor {{actor_id}}",
                         ),
                     ],
                 ),
@@ -3835,7 +3848,7 @@ def section_iceberg_metrics(outer_panels):
                     [
                         panels.target(
                             f"{metric('iceberg_position_delete_cache_num')}",
-                            "{{executor_id}} @ {{sink_id}}",
+                            "{{sink_id}} {{sink_name}} actor {{actor_id}}",
                         ),
                     ],
                 ),
@@ -3845,7 +3858,7 @@ def section_iceberg_metrics(outer_panels):
                     [
                         panels.target(
                             f"{metric('iceberg_partition_num')}",
-                            "{{executor_id}} @ {{sink_id}}",
+                            "{{sink_id}} {{sink_name}} actor {{actor_id}}",
                         ),
                     ],
                 ),
@@ -4012,14 +4025,14 @@ def section_sink_metrics(outer_panels):
                     [
                         *quantile(
                             lambda quantile, legend: panels.target(
-                                f"histogram_quantile({quantile}, sum(rate({metric('sink_commit_duration_bucket')}[$__rate_interval])) by (le, connector, sink_id))",
-                                f"p{legend}" + " @ {{connector}} {{sink_id}}",
+                                f"histogram_quantile({quantile}, sum(rate({metric('sink_commit_duration_bucket')}[$__rate_interval])) by (le, connector, sink_id, sink_name))",
+                                f"p{legend}" + " @ {{sink_id}} {{sink_name}} ({{connector}})",
                             ),
                             [50, 99, "max"],
                         ),
                         panels.target(
-                            f"sum by(le, connector, sink_id)(rate({metric('sink_commit_duration_sum')}[$__rate_interval])) / sum by(le, type, {COMPONENT_LABEL}, {NODE_LABEL}) (rate({metric('sink_commit_duration_count')}[$__rate_interval])) > 0",
-                            "avg - {{connector}} @ {{sink_id}}",
+                            f"sum by(le, type, {COMPONENT_LABEL}, {NODE_LABEL}, sink_id, sink_name)(rate({metric('sink_commit_duration_sum')}[$__rate_interval])) / sum by(le, type, {COMPONENT_LABEL}, {NODE_LABEL}, sink_id, sink_name) (rate({metric('sink_commit_duration_count')}[$__rate_interval])) > 0",
+                            "avg @ {{sink_id}} {{sink_name}} ({{connector}})",
                         ),
                     ],
                 ),
@@ -4029,15 +4042,15 @@ def section_sink_metrics(outer_panels):
                     [
                         panels.target(
                             f"{metric('log_store_latest_write_epoch')}",
-                            "latest write epoch @ {{connector}} {{sink_id}} {{executor_id}}",
+                            "latest write epoch @ {{sink_id}} {{sink_name}} ({{connector}}) actor {{actor_id}}",
                         ),
                         panels.target(
                             f"{metric('log_store_latest_read_epoch')}",
-                            "latest read epoch @ {{connector}} {{sink_id}} {{executor_id}}",
+                            "latest read epoch @ {{sink_id}} {{sink_name}} ({{connector}}) actor {{actor_id}}",
                         ),
                         panels.target(
                             f"{metric('kv_log_store_buffer_unconsumed_min_epoch')}",
-                            "Kv log store uncomsuned min epoch @ {{connector}} {{sink_id}} {{executor_id}}",
+                            "Kv log store uncomsuned min epoch @ {{sink_id}} {{sink_name}} ({{connector}}) actor {{actor_id}}",
                         ),
                     ],
                 ),
@@ -4046,9 +4059,9 @@ def section_sink_metrics(outer_panels):
                     "",
                     [
                         panels.target(
-                            f"(max({metric('log_store_latest_write_epoch')}) by (connector, sink_id, executor_id)"
-                            + f"- max({metric('log_store_latest_read_epoch')}) by (connector, sink_id, executor_id)) / (2^16) / 1000",
-                            "Consume lag @ {{connector}} {{sink_id}} {{executor_id}}",
+                            f"(max({metric('log_store_latest_write_epoch')}) by (connector, sink_id, actor_id, sink_name)"
+                            + f"- max({metric('log_store_latest_read_epoch')}) by (connector, sink_id, actor_id, sink_name)) / (2^16) / 1000",
+                            "Consume lag @ {{sink_id}} {{sink_name}} ({{connector}}) actor {{actor_id}}",
                         ),
                     ],
                 ),
@@ -4057,8 +4070,8 @@ def section_sink_metrics(outer_panels):
                     "",
                     [
                         panels.target(
-                            f"avg(rate({metric('log_store_reader_wait_new_future_duration_ns')}[$__rate_interval])) by (connector, sink_id, executor_id) / 1000000000",
-                            "Backpressure @ {{connector}} {{sink_id}} {{executor_id}}",
+                            f"avg(rate({metric('log_store_reader_wait_new_future_duration_ns')}[$__rate_interval])) by (connector, sink_id, actor_id, sink_name) / 1000000000",
+                            "Backpressure @ {{sink_id}} {{sink_name}} ({{connector}}) actor {{actor_id}}",
                         ),
                     ],
                 ),
@@ -4067,9 +4080,9 @@ def section_sink_metrics(outer_panels):
                     "",
                     [
                         panels.target(
-                            f"clamp_min((max({metric('log_store_first_write_epoch')}) by (connector, sink_id, executor_id)"
-                            + f"- max({metric('log_store_latest_read_epoch')}) by (connector, sink_id, executor_id)) / (2^16) / 1000, 0)",
-                            "Consume persistent log lag @ {{connector}} {{sink_id}} {{executor_id}}",
+                            f"clamp_min((max({metric('log_store_first_write_epoch')}) by (connector, sink_id, actor_id, sink_name)"
+                            + f"- max({metric('log_store_latest_read_epoch')}) by (connector, sink_id, actor_id, sink_name)) / (2^16) / 1000, 0)",
+                            "Consume persistent log lag @ {{sink_id}} {{sink_name}} ({{connector}}) actor {{actor_id}}",
                         ),
                     ],
                 ),
@@ -4078,8 +4091,8 @@ def section_sink_metrics(outer_panels):
                     "",
                     [
                         panels.target(
-                            f"sum(rate({metric('log_store_read_rows')}[$__rate_interval])) by (connector, sink_id)",
-                            "sink={{connector}} {{sink_id}}",
+                            f"sum(rate({metric('log_store_read_rows')}[$__rate_interval])) by (connector, sink_id, sink_name)",
+                            "{{sink_id}} {{sink_name}} ({{connector}})",
                         ),
                     ],
                 ),
@@ -4088,8 +4101,8 @@ def section_sink_metrics(outer_panels):
                     "",
                     [
                         panels.target(
-                            f"sum(rate({metric('log_store_read_rows')}[$__rate_interval])) by ({NODE_LABEL}, connector, sink_id, executor_id)",
-                            "sink={{connector}} {{sink_id}} @ {{executor_id}} {{%s}}"
+                            f"sum(rate({metric('log_store_read_rows')}[$__rate_interval])) by ({NODE_LABEL}, connector, sink_id, actor_id, sink_name)",
+                            "{{sink_id}} {{sink_name}} ({{connector}}) actor {{actor_id}} @ {{%s}}"
                             % NODE_LABEL,
                         ),
                     ],
@@ -4099,8 +4112,8 @@ def section_sink_metrics(outer_panels):
                     "",
                     [
                         panels.target(
-                            f"sum(rate({metric('log_store_write_rows')}[$__rate_interval])) by (connector, sink_id)",
-                            "sink={{connector}} {{sink_id}}",
+                            f"sum(rate({metric('log_store_write_rows')}[$__rate_interval])) by (connector, sink_id, sink_name)",
+                            "sink={{sink_id}} {{sink_name}} ({{connector}})",
                         ),
                     ],
                 ),
@@ -4109,8 +4122,8 @@ def section_sink_metrics(outer_panels):
                     "",
                     [
                         panels.target(
-                            f"sum(rate({metric('log_store_write_rows')}[$__rate_interval])) by ({NODE_LABEL}, connector, sink_id, executor_id)",
-                            "sink={{connector}} {{sink_id}} @ {{executor_id}} {{%s}}"
+                            f"sum(rate({metric('log_store_write_rows')}[$__rate_interval])) by ({NODE_LABEL}, connector, sink_id, actor_id, sink_name)",
+                            "{{sink_id}} {{sink_name}} ({{connector}}) actor {{actor_id}} {{%s}}"
                             % NODE_LABEL,
                         ),
                     ],
@@ -4120,8 +4133,8 @@ def section_sink_metrics(outer_panels):
                     "",
                     [
                         panels.target(
-                            f"sum(rate({metric('kv_log_store_storage_read_count')}[$__rate_interval])) by (executor_id, connector, sink_id)",
-                            "{{executor_id}} - {{connector}} @ {{sink_id}}",
+                            f"sum(rate({metric('kv_log_store_storage_read_count')}[$__rate_interval])) by (actor_id, connector, sink_id, sink_name)",
+                            "{{sink_id}} {{sink_name}} actor {{actor_id}} ({{connector}})",
                         ),
                     ],
                 ),
@@ -4130,8 +4143,8 @@ def section_sink_metrics(outer_panels):
                     "",
                     [
                         panels.target(
-                            f"sum(rate({metric('kv_log_store_storage_read_size')}[$__rate_interval])) by (executor_id, connector, sink_id)",
-                            "{{executor_id}} - {{connector}} @ {{sink_id}}",
+                            f"sum(rate({metric('kv_log_store_storage_read_size')}[$__rate_interval])) by (actor_id, connector, sink_id, sink_name)",
+                            "{{sink_id}} {{sink_name}} actor {{actor_id}} ({{connector}})",
                         ),
                     ],
                 ),
@@ -4140,8 +4153,8 @@ def section_sink_metrics(outer_panels):
                     "",
                     [
                         panels.target(
-                            f"sum(rate({metric('kv_log_store_storage_write_count')}[$__rate_interval])) by (executor_id, connector, sink_id)",
-                            "{{executor_id}} - {{connector}} @ {{sink_id}}",
+                            f"sum(rate({metric('kv_log_store_storage_write_count')}[$__rate_interval])) by (actor_id, connector, sink_id, sink_name)",
+                            "{{sink_id}} {{sink_name}} actor {{actor_id}} ({{connector}})",
                         ),
                     ],
                 ),
@@ -4150,8 +4163,8 @@ def section_sink_metrics(outer_panels):
                     "",
                     [
                         panels.target(
-                            f"sum(rate({metric('kv_log_store_storage_write_size')}[$__rate_interval])) by (executor_id, connector, sink_id)",
-                            "{{executor_id}} - {{connector}} @ {{sink_id}}",
+                            f"sum(rate({metric('kv_log_store_storage_write_size')}[$__rate_interval])) by (actor_id, connector, sink_id, sink_name)",
+                            "{{sink_id}} {{sink_name}} actor {{actor_id}} ({{connector}})",
                         ),
                     ],
                 ),
@@ -4161,15 +4174,15 @@ def section_sink_metrics(outer_panels):
                     [
                         panels.target(
                             f"{metric('kv_log_store_buffer_unconsumed_item_count')}",
-                            "Unconsumed item count @ {{connector}} {{sink_id}} {{executor_id}}",
+                            "Unconsumed item count @ {{sink_id}} {{sink_name}} ({{connector}}) actor {{actor_id}}",
                         ),
                         panels.target(
                             f"{metric('kv_log_store_buffer_unconsumed_row_count')}",
-                            "Unconsumed row count @ {{connector}} {{sink_id}} {{executor_id}}",
+                            "Unconsumed row count @ {{sink_id}} {{sink_name}} ({{connector}}) actor {{actor_id}}",
                         ),
                         panels.target(
                             f"{metric('kv_log_store_buffer_unconsumed_epoch_count')}",
-                            "Unconsumed epoch count @ {{connector}} {{sink_id}} {{executor_id}}",
+                            "Unconsumed epoch count @ {{sink_id}} {{sink_name}} ({{connector}}) actor {{actor_id}}",
                         ),
                     ],
                 ),
@@ -4178,8 +4191,8 @@ def section_sink_metrics(outer_panels):
                     "",
                     [
                         panels.target(
-                            f"sum(rate({metric('kv_log_store_rewind_count')}[$__rate_interval])) by (executor_id, connector, sink_id)",
-                            "{{executor_id}} - {{connector}} @ {{sink_id}}",
+                            f"sum(rate({metric('kv_log_store_rewind_count')}[$__rate_interval])) by (actor_id, connector, sink_id, sink_name)",
+                            "{{sink_id}} {{sink_name}} actor {{actor_id}} ({{connector}})",
                         ),
                     ],
                 ),
@@ -4188,8 +4201,8 @@ def section_sink_metrics(outer_panels):
                     "",
                     [
                         panels.target(
-                            f"histogram_quantile(1.0, sum(rate({metric('kv_log_store_rewind_delay_bucket')}[$__rate_interval])) by (le, executor_id, connector, sink_id))",
-                            "{{executor_id}} - {{connector}} @ {{sink_id}}",
+                            f"histogram_quantile(1.0, sum(rate({metric('kv_log_store_rewind_delay_bucket')}[$__rate_interval])) by (le, actor_id, connector, sink_id, sink_name))",
+                            "{{sink_id}} {{sink_name}} actor {{actor_id}} ({{connector}})",
                         ),
                     ],
                 ),
@@ -4198,7 +4211,7 @@ def section_sink_metrics(outer_panels):
                     "Total size of chunks buffered in a barrier",
                     [
                         panels.target(
-                            f"sum({metric('stream_sink_chunk_buffer_size')}) by (sink_id, actor_id) * on(actor_id) group_left(sink_name) {metric('sink_info')}",
+                            f"sum({metric('stream_sink_chunk_buffer_size')}) by (sink_id, actor_id, sink_name) * on(actor_id) group_left(sink_name) {metric('sink_info')}",
                             "sink {{sink_id}} {{sink_name}} - actor {{actor_id}}",
                         ),
                     ],
diff --git a/grafana/risingwave-dev-dashboard.json b/grafana/risingwave-dev-dashboard.json
index 3e303bca36ae2..1a14442fc9fe0 100644
--- a/grafana/risingwave-dev-dashboard.json
+++ b/grafana/risingwave-dev-dashboard.json
@@ -1 +1 @@
-{"__inputs":[],"annotations":{"list":[]},"description":"RisingWave Dev Dashboard","editable":true,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"panels":[{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":1,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Information about actors","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":2,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true,"sortBy":[]},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"group(actor_info{job=~\"$job\",instance=~\"$node\"}) by (actor_id, fragment_id, compute_node)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"group(actor_info{job=~\"$job\",instance=~\"$node\"}) by (actor_id, fragment_id, compute_node)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true},"indexByName":{"actor_id":0,"compute_node":2,"fragment_id":1}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Information about state tables. Column `materialized_view_id` is the id of the materialized view that this state table belongs to.","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":3,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true,"sortBy":[]},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name, table_type, materialized_view_id, fragment_id, compaction_group_id)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name, table_type, materialized_view_id, fragment_id, compaction_group_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"State Table Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true},"indexByName":{"compaction_group_id":5,"fragment_id":4,"materialized_view_id":3,"table_id":0,"table_name":1,"table_type":2}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Actor count per compute node","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":4,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true,"sortBy":[]},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"count(actor_info{job=~\"$job\",instance=~\"$node\"}) by (compute_node)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"count(actor_info{job=~\"$job\",instance=~\"$node\"}) by (compute_node)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Count (Group By Compute Node)","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true},"indexByName":{"compaction_group_id":5,"fragment_id":4,"materialized_view_id":3,"table_id":0,"table_name":1,"table_type":2}}}],"transparent":false,"type":"table"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Actor/Table Id Info","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":1},"height":null,"hideTimeOverride":false,"id":5,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of each type of RisingWave components alive.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":6,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_type}}","metric":"","query":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The memory usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":7,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Memory","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Memory usage relative to k8s resource limit of container. Only works in K8s environment","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":8,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(avg by(namespace, pod) (container_memory_working_set_bytes{namespace=~\"$namespace\",pod=~\"$pod\",container=~\"$component\"})) / (  sum by(namespace, pod) (kube_pod_container_resource_limits{namespace=~\"$namespace\", pod=~\"$pod\", container=\"$component\", resource=\"memory\", unit=\"byte\"}))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg memory usage @ {{job}} @ {{instance}}","metric":"","query":"(avg by(namespace, pod) (container_memory_working_set_bytes{namespace=~\"$namespace\",pod=~\"$pod\",container=~\"$component\"})) / (  sum by(namespace, pod) (kube_pod_container_resource_limits{namespace=~\"$namespace\", pod=~\"$pod\", container=\"$component\", resource=\"memory\", unit=\"byte\"}))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Memory relative","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The CPU usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":9,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cpu usage (total) - {{job}} @ {{instance}}","metric":"","query":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (job, instance) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cpu usage (avg per core) - {{job}} @ {{instance}}","metric":"","query":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (job, instance) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"CPU usage relative to k8s resource limit of container. Only works in K8s environment","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":10,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\",container=~\"$component\",pod=~\"$pod\"}[$__rate_interval])) by (namespace, pod)) / (sum(kube_pod_container_resource_limits{namespace=~\"$namespace\",pod=~\"$pod\",container=~\"$component\", resource=\"cpu\"}) by (namespace, pod))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cpu usage @ {{job}} @ {{instance}}","metric":"","query":"(sum(rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\",container=~\"$component\",pod=~\"$pod\"}[$__rate_interval])) by (namespace, pod)) / (sum(kube_pod_container_resource_limits{namespace=~\"$namespace\",pod=~\"$pod\",container=~\"$component\", resource=\"cpu\"}) by (namespace, pod))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU relative","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"RW cluster can configure multiple meta nodes to achieve high availability. One is the leader and the rest are the followers.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":11,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(meta_num{job=~\"$job\",instance=~\"$node\"}) by (worker_addr,role)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_addr}} @ {{role}}","metric":"","query":"sum(meta_num{job=~\"$job\",instance=~\"$node\"}) by (worker_addr,role)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Meta Cluster","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Cluster Node","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":2},"height":null,"hideTimeOverride":false,"id":12,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The rate of successful recovery attempts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":13,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Recovery Successful Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of failed reocovery attempts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":14,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Failed recovery attempts","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Time spent in a successful recovery attempt","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":15,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency pmax - {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by (le) (rate(recovery_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by (le) (rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency avg","metric":"","query":"sum by (le) (rate(recovery_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by (le) (rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Recovery latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Recovery","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":3},"height":null,"hideTimeOverride":false,"id":16,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of barriers that have been ingested but not completely processed. This metric reflects the current level of congestion within the system.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":17,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all_barrier","metric":"","query":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"in_flight_barrier_nums{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"in_flight_barrier","metric":"","query":"in_flight_barrier_nums{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The time that the data between two consecutive barriers gets fully processed, i.e. the computation results are made durable into materialized views or sink to external systems. This metric shows to users the freshness of materialized views.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":18,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_avg","metric":"","query":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The duration from the last committed barrier's epoch time to the current time. This metric reflects the data freshness of the system. During this time, no new data has been committed.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":19,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"timestamp(last_committed_barrier_time{job=~\"$job\",instance=~\"$node\"}) - last_committed_barrier_time{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_pending_time","metric":"","query":"timestamp(last_committed_barrier_time{job=~\"$job\",instance=~\"$node\"}) - last_committed_barrier_time{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier pending time (secs)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":20,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","query":"sum(rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Each query is executed in parallel with a user-defined parallelism. This figure shows the throughput of each parallelism. The throughput of all the parallelism added up is equal to Source Throughput(rows).","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":21,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(source_partition_input_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor={{actor_id}} source={{source_id}} partition={{partition}} fragment_id={{fragment_id}}","metric":"","query":"rate(source_partition_input_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s) Per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of bytes read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":22,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum by (source_id, source_name, fragment_id)(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","query":"(sum by (source_id, source_name, fragment_id)(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Each query is executed in parallel with a user-defined parallelism. This figure shows the throughput of each parallelism. The throughput of all the parallelism added up is equal to Source Throughput(MB/s).","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":23,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor={{actor_id}} source={{source_id}} partition={{partition}} fragment_id={{fragment_id}}","metric":"","query":"(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))/(1000*1000)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s) Per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":24,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_source_backfill_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","query":"sum(rate(stream_source_backfill_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Backfill Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Monitor each source upstream, 0 means the upstream is not normal, 1 means the source is ready.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":25,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_status_is_up{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source_id={{source_id}}, source_name={{source_name}} @ {{instance}}","metric":"","query":"source_status_is_up{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Upstream Status","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Source Split Change Events frequency by source_id and actor_id","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":26,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_source_split_change_event_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_name}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_source_split_change_event_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Split Change Events frequency(events/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Kafka Consumer Lag Size by source_id, partition and actor_id","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":27,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_min(source_kafka_high_watermark{job=~\"$job\",instance=~\"$node\"} - on(source_id, partition) group_right() source_latest_message_id{job=~\"$job\",instance=~\"$node\"}, 0)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_id}} partition={{partition}} actor_id={{actor_id}}","metric":"","query":"clamp_min(source_kafka_high_watermark{job=~\"$job\",instance=~\"$node\"} - on(source_id, partition) group_right() source_latest_message_id{job=~\"$job\",instance=~\"$node\"}, 0)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kafka Consumer Lag Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of rows streamed into each sink per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":28,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id) * on(sink_id) group_left(sink_name) group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}}","metric":"","query":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id) * on(sink_id) group_left(sink_name) group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (sink_id, sink_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of rows streamed into each sink per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":29,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id, actor_id) * on(actor_id) group_left(sink_name) sink_info{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}} - actor {{actor_id}}","metric":"","query":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id, actor_id) * on(actor_id) group_left(sink_name) sink_info{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Throughput(rows/s) per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized view per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":30,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id) * on(table_id) group_left(table_name) group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mview {{table_id}} {{table_name}}","metric":"","query":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id) * on(table_id) group_left(table_name) group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized view per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":31,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) * on(fragment_id, table_id) group_left(table_name) table_info{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mview {{table_id}} {{table_name}} - actor {{actor_id}} fragment_id {{fragment_id}}","metric":"","query":"rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) * on(fragment_id, table_id) group_left(table_name) table_info{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s) per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been read from the backfill snapshot","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":32,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_snapshot_read_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_backfill_snapshot_read_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Snapshot Read Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been output from the backfill upstream","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":33,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_upstream_output_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_backfill_upstream_output_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Upstream Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The duration between the time point when the scheduled barrier needs to be sent and the time point when the barrier gets actually sent to all the compute nodes. Developers can thus detect any internal congestion.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":34,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_send_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_send_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_avg","metric":"","query":"rate(meta_barrier_send_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_send_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Send Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":35,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"max(sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_avg","metric":"","query":"max(sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier In-Flight Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":36,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p999 - {{instance}}","metric":"","query":"histogram_quantile(0.999, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_pmax - {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_avg - {{instance}}","metric":"","query":"sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Sync Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":37,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_wait_commit_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_wait_commit_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_avg","metric":"","query":"rate(meta_barrier_wait_commit_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_wait_commit_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Wait Commit Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of actors that have processed the earliest in-flight barriers per second. This metric helps users to detect potential congestion or stuck in the system.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":38,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_barrier_manager_progress{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"rate(stream_barrier_manager_progress{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Earliest In-Flight Barrier Progress","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":4},"height":null,"hideTimeOverride":false,"id":39,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been read from the cdc backfill snapshot","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":40,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_cdc_backfill_snapshot_read_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_cdc_backfill_snapshot_read_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CDC Backfill Snapshot Read Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been output from the cdc backfill upstream","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":41,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_cdc_backfill_upstream_output_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_cdc_backfill_upstream_output_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CDC Backfill Upstream Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":42,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lag p50 - {{table_name}}","metric":"","query":"histogram_quantile(0.5, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lag p99 - {{table_name}}","metric":"","query":"histogram_quantile(0.99, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lag pmax - {{table_name}}","metric":"","query":"histogram_quantile(1.0, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CDC Consume Lag Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":43,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(cdc_source_error{job=~\"$job\",instance=~\"$node\"}) by (connector_name, source_id, error_msg)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{connector_name}}: {{error_msg}} ({{source_id}})","metric":"","query":"sum(cdc_source_error{job=~\"$job\",instance=~\"$node\"}) by (connector_name, source_id, error_msg)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CDC Source Errors","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming CDC","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":5},"height":null,"hideTimeOverride":false,"id":44,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"We first record the total blocking duration(ns) of output buffer of each actor. It shows how much time it takes an actor to process a message, i.e. a barrier, a watermark or rows of data, on average. Then we divide this duration by 1 second and show it as a percentage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":45,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, downstream_fragment_id) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}->{{downstream_fragment_id}}","metric":"","query":"avg(rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, downstream_fragment_id) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Output Blocking Time Ratio (Backpressure)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":46,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_actor_input_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, upstream_fragment_id) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}<-{{upstream_fragment_id}}","metric":"","query":"avg(rate(stream_actor_input_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, upstream_fragment_id) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Input Blocking Time Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":47,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_actor_in_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, upstream_fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}<-{{upstream_fragment_id}}","metric":"","query":"sum(rate(stream_actor_in_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, upstream_fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_in_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}}","metric":"","query":"rate(stream_actor_in_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Input Throughput (rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":48,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_actor_out_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_actor_out_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_out_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}}","metric":"","query":"rate(stream_actor_out_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Output Throughput (rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The operator-level memory usage statistics collected by each LRU cache","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":49,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (table_id, desc)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table {{table_id}} desc: {{desc}}","metric":"","query":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (table_id, desc)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_memory_usage{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table {{table_id}} actor {{actor_id}} desc: {{desc}}","metric":"","query":"stream_memory_usage{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Memory Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Memory usage aggregated by materialized views","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":50,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id) group_left(materialized_view_id) table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized view {{materialized_view_id}}","metric":"","query":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id) group_left(materialized_view_id) table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Memory Usage of Materialized Views","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":51,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"temporal join cache miss, table_id {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"temporal join cache miss, table_id {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Temporal Join Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":52,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_materialize_cache_hit_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache hit count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_materialize_cache_hit_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_materialize_cache_total_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total cached count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_materialize_cache_total_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_hit_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache hit count - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_materialize_cache_hit_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_total_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total cached count - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_materialize_cache_total_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialize Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":53,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache lookup count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_over_window_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_cache_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache miss count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_over_window_cache_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_over_window_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache lookup count - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_over_window_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_over_window_cache_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache miss count - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_over_window_cache_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_range_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"partition range cache lookup count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_over_window_range_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_range_cache_left_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"partition range cache left miss count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_over_window_range_cache_left_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_range_cache_right_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"partition range cache right miss count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_over_window_range_cache_right_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Over Window Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":54,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, fragment_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Join executor cache miss ratio - - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} fragment {{fragment_id}}","metric":"","query":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, fragment_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n appendonly cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream lookup cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream temporal join cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Materialize executor cache miss ratio - table {{table_id}} fragment {{fragment_id}}  {{instance}}","metric":"","query":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_over_window_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Over window cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_over_window_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_over_window_range_cache_left_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_range_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Over window partition range cache left miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_over_window_range_cache_left_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_range_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_over_window_range_cache_right_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_range_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Over window partition range cache right miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_over_window_range_cache_right_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_range_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Miss Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":55,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, executor, fragment_id, wait_side, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - executor {{executor}} fragment {{fragment_id}} {{wait_side}} - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, executor, fragment_id, wait_side, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, executor, fragment_id, wait_side, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - executor {{executor}} fragment {{fragment_id}} {{wait_side}} - {{job}}","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, executor, fragment_id, wait_side, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, executor, fragment_id, wait_side, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p999 - executor {{executor}} fragment {{fragment_id}} {{wait_side}} - {{job}}","metric":"","query":"histogram_quantile(0.999, sum(rate(stream_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, executor, fragment_id, wait_side, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, executor, fragment_id, wait_side, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - executor {{executor}} fragment {{fragment_id}} {{wait_side}} - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, executor, fragment_id, wait_side, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, executor, fragment_id, wait_side, job)(rate(stream_barrier_align_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le,executor,fragment_id,wait_side,job) (rate(stream_barrier_align_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - executor {{executor}} fragment {{fragment_id}} {{wait_side}} - {{job}}","metric":"","query":"sum by(le, executor, fragment_id, wait_side, job)(rate(stream_barrier_align_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le,executor,fragment_id,wait_side,job) (rate(stream_barrier_align_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Barrier Align","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":56,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - fragment {{fragment_id}} - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - fragment {{fragment_id}} - {{job}}","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p999 - fragment {{fragment_id}} - {{job}}","metric":"","query":"histogram_quantile(0.999, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - fragment {{fragment_id}} - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, fragment_id, job)(rate(stream_merge_barrier_align_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le,fragment_id,job) (rate(stream_merge_barrier_align_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - fragment {{fragment_id}} - {{job}}","metric":"","query":"sum by(le, fragment_id, job)(rate(stream_merge_barrier_align_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le,fragment_id,job) (rate(stream_merge_barrier_align_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Merger Barrier Align","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":57,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_join_actor_input_waiting_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000) by (fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}","metric":"","query":"avg(rate(stream_join_actor_input_waiting_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000) by (fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_actor_input_waiting_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}}","metric":"","query":"rate(stream_join_actor_input_waiting_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Actor Input Blocking Time Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":58,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_join_match_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000) by (fragment_id,side)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}} {{side}}","metric":"","query":"avg(rate(stream_join_match_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000) by (fragment_id,side)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_match_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}} {{side}}","metric":"","query":"rate(stream_join_match_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Actor Match Duration Per Second","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Multiple rows with distinct primary keys may have the same join key. This metric counts the number of join keys in the executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":59,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (fragment_id, side)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}} {{side}}","metric":"","query":"sum(stream_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (fragment_id, side)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}} {{side}}","metric":"","query":"stream_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of matched rows on the opposite side","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":60,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - fragment {{fragment_id}} table_id {{table_id}} - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - fragment {{fragment_id}} table_id {{table_id}} - {{job}}","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - fragment {{fragment_id}} table_id {{table_id}} - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, fragment_id, table_id) (rate(stream_join_matched_join_keys_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, fragment_id, table_id) (rate(stream_join_matched_join_keys_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - fragment {{fragment_id}} table_id {{table_id}} - {{job}}","metric":"","query":"sum by(le, job, fragment_id, table_id) (rate(stream_join_matched_join_keys_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, fragment_id, table_id) (rate(stream_join_matched_join_keys_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) >= 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Executor Matched Rows","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":61,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_agg_chunk_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level cache miss - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_agg_chunk_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_agg_chunk_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level total lookups - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_agg_chunk_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_chunk_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level cache miss  - table {{table_id}} actor {{actor_id}}}","metric":"","query":"rate(stream_agg_chunk_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_chunk_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level total lookups  - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_chunk_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Executor Cache Statistics For Each StreamChunk","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in each hash aggregation executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":62,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_agg_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg cached keys count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_agg_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_agg_distinct_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg distinct cached keys count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_agg_distinct_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg cached keys count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_agg_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_distinct_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg distinct cached keys count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_agg_distinct_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of dirty (unflushed) groups in each hash aggregation executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":63,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_agg_dirty_groups_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_agg_dirty_groups_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_dirty_groups_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_agg_dirty_groups_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Dirty Groups Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The total heap size of dirty (unflushed) groups in each hash aggregation executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":64,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_agg_dirty_groups_heap_size{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups heap size | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_agg_dirty_groups_heap_size{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_dirty_groups_heap_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups heap size | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_agg_dirty_groups_heap_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Dirty Groups Heap Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in each top_n executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":65,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_group_top_n_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_group_top_n_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_group_top_n_appendonly_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n appendonly cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_group_top_n_appendonly_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_group_top_n_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_group_top_n_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_group_top_n_appendonly_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n appendonly cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_group_top_n_appendonly_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"TopN Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in temporal join executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":66,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_temporal_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal Join cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_temporal_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_temporal_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal Join cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_temporal_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Temporal Join Cache Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in lookup executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":88},"height":null,"hideTimeOverride":false,"id":67,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_lookup_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lookup cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_lookup_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_lookup_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lookup cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_lookup_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lookup Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in over window executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":88},"height":null,"hideTimeOverride":false,"id":68,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_over_window_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"over window cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_over_window_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_over_window_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"over window cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_over_window_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_over_window_range_cache_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"over window partition range cache entry count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_over_window_range_cache_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Over Window Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"When enabled, this metric shows the input throughput of each executor.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":96},"height":null,"hideTimeOverride":false,"id":69,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_executor_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_identity, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_identity}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_executor_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_identity, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_executor_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_identity}} actor {{actor_id}}","metric":"","query":"rate(stream_executor_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The actor-level memory usage statistics reported by TaskLocalAlloc. (Disabled by default)","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":96},"height":null,"hideTimeOverride":false,"id":70,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(actor_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}","metric":"","query":"sum(actor_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"actor_memory_usage{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}}","metric":"","query":"actor_memory_usage{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Memory Usage (TaskLocalAlloc)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Actors","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":6},"height":null,"hideTimeOverride":false,"id":71,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":72,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_actor_execution_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_actor_execution_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Execution Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":73,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":74,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":8},"height":null,"hideTimeOverride":false,"id":75,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":8},"height":null,"hideTimeOverride":false,"id":76,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":77,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":16},"height":null,"hideTimeOverride":false,"id":78,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":16},"height":null,"hideTimeOverride":false,"id":79,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":80,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":24},"height":null,"hideTimeOverride":false,"id":81,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":24},"height":null,"hideTimeOverride":false,"id":82,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":83,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":32},"height":null,"hideTimeOverride":false,"id":84,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":32},"height":null,"hideTimeOverride":false,"id":85,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":86,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":40},"height":null,"hideTimeOverride":false,"id":87,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Avg Time","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Actors (Tokio)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":7},"height":null,"hideTimeOverride":false,"id":88,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":89,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{up_fragment_id}}->{{down_fragment_id}}","metric":"","query":"rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fragment-level Remote Exchange Send Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":90,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{up_fragment_id}}->{{down_fragment_id}}","metric":"","query":"rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fragment-level Remote Exchange Recv Throughput","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Exchange","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":91,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Errors that happened during computation. Check the logs for detailed error message.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":92,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_compute_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, executor_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{executor_name}} (fragment_id={{fragment_id}})","metric":"","query":"sum(user_compute_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, executor_name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compute Errors by Type","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Errors that happened during source data ingestion. Check the logs for detailed error message.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":93,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_source_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{source_name}} (source_id={{source_id}} fragment_id={{fragment_id}})","metric":"","query":"sum(user_source_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, source_id, source_name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Errors by Type","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Errors that happened during data sink out. Check the logs for detailed error message.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":94,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_sink_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, sink_id, sink_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{sink_name}} (sink_id={{sink_id}} fragment_id={{fragment_id}})","metric":"","query":"sum(user_sink_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, sink_id, sink_name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Errors by Type","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"User Streaming Errors","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":9},"height":null,"hideTimeOverride":false,"id":95,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"row"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":96,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{query_id}} : {{source_stage_id}}.{{source_task_id}} -> {{target_stage_id}}.{{target_task_id}}","metric":"","query":"batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Exchange Recv Row Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":97,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_task_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"batch_task_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Mpp Task Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"All memory usage of batch executors in bytes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":98,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"compute_batch_total_mem{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"compute_batch_total_mem{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"frontend_batch_total_mem{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"frontend_batch_total_mem{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Mem Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":99,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_heartbeat_worker_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"batch_heartbeat_worker_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Heartbeat Worker Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":100,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Row SeqScan Next Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Disk throughputs of spilling-out in the bacth query engine","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":101,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(batch_spill_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}} @ {{instance}}","metric":"","query":"sum(rate(batch_spill_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(batch_spill_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}} @ {{instance}}","metric":"","query":"sum(rate(batch_spill_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Spill Throughput","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Batch Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":10},"height":null,"hideTimeOverride":false,"id":102,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":103,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{table_id}} @ {{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_sst_store_block_request_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total_meta_miss_count - {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Hummock has three parts of memory usage: 1. Meta Cache 2. Block CacheThis metric shows the real memory usage of each of these three caches.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":104,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta cache - {{job}} @ {{instance}}","metric":"","query":"avg(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"data cache - {{job}} @ {{instance}}","metric":"","query":"avg(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(state_store_prefetch_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"prefetch cache - {{job}} @ {{instance}}","metric":"","query":"avg(state_store_prefetch_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":105,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='meta_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta cache miss ratio - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"(sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='meta_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_sst_store_block_request_counts{type='data_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='data_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache miss ratio - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"(sum(rate(state_store_sst_store_block_request_counts{type='data_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='data_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) >= 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Miss Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of the estimated hit ratio of a block while in the block cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":106,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_max(histogram_quantile(0.1, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache efficienfy - p10 - {{job}} @ {{instance}}","metric":"","query":"clamp_max(histogram_quantile(0.1, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_max(histogram_quantile(0.25, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache efficienfy - p25 - {{job}} @ {{instance}}","metric":"","query":"clamp_max(histogram_quantile(0.25, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_max(histogram_quantile(0.5, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache efficienfy - p50 - {{job}} @ {{instance}}","metric":"","query":"clamp_max(histogram_quantile(0.5, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_max(histogram_quantile(0.75, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache efficienfy - p75 - {{job}} @ {{instance}}","metric":"","query":"clamp_max(histogram_quantile(0.75, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_max(histogram_quantile(0.9, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache efficienfy - p90 - {{job}} @ {{instance}}","metric":"","query":"clamp_max(histogram_quantile(0.9, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_max(histogram_quantile(1.0, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache efficienfy - p100 - {{job}} @ {{instance}}","metric":"","query":"clamp_max(histogram_quantile(1.0, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Block Cache Efficiency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":107,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_scan_key_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type, table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter keys flow - {{table_id}} @ {{type}} @ {{instance}}","metric":"","query":"sum(rate(state_store_iter_scan_key_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type, table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iter keys flow","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":108,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts p50 - {{table_id}} @ {{job}} @ {{type}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts p99 - {{table_id}} @ {{job}} @ {{type}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts pmax - {{table_id}} @ {{job}} @ {{type}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts avg  - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Merged SSTs","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of the latency of Get operations that have been issued to the state store.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":109,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_get_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, table_id)(rate(state_store_get_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Duration - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of the time spent on iterator initialization.Histogram of the time spent on iterator scanning.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":110,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time p50 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time p99 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time pmax - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_iter_init_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, iter_type) (rate(state_store_iter_init_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time avg - {{iter_type}} {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(state_store_iter_init_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, iter_type) (rate(state_store_iter_init_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time p50 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time p99 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time pmax - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_iter_scan_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, iter_type) (rate(state_store_iter_scan_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time avg - {{iter_type}} {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(state_store_iter_scan_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, iter_type) (rate(state_store_iter_scan_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Duration - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":111,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter false positive count  - {{table_id}} - {{type}}","metric":"","query":"sum(irate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter positive count - {{table_id}} - {{type}}","metric":"","query":"sum(irate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter check count- {{table_id}} - {{type}}","metric":"","query":"sum(irate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom Filter Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Positive / Total","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":112,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter positive rate - {{table_id}} - {{type}}","metric":"","query":"(sum(rate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)) >= 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom Filter Positive Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"False-Positive / Total","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":113,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(((sum(rate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read req bloom filter false positive rate - {{table_id}} - {{type}}","metric":"","query":"(((sum(rate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))) >= 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom Filter False-Positive Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":114,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_iter_slow_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"state_store_iter_slow_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Slow Fetch Meta Unhits","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":115,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_shared_buffer_hit_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"shared_buffer hit - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_get_shared_buffer_hit_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id, iter_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{iter_type}} - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_iter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id, iter_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":116,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Size - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":117,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Size - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":118,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read p50 - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read p99 - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read pmax - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Read Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":119,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_iter_in_progress_counts{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Existing {{iter_type}} count @ {{table_id}}","metric":"","query":"state_store_iter_in_progress_counts{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_log_op_type_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, op_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter_log op count @ {{table_id}} {{op_type}}","metric":"","query":"sum(rate(state_store_iter_log_op_type_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, op_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Count - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size of a single key-value pair when reading by operation Get.Operation Get gets a single key-value pair with respect to a caller-specified key. If the key does not exist in the storage, the size of key is counted into this metric and the size of value is 0.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":120,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance) + sum(rate(state_store_get_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_get_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance) + sum(rate(state_store_get_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Throughput - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size of all the key-value paris when reading by operation Iter.Operation Iter scans a range of key-value pairs.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":121,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_iter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Throughput - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":122,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration avg - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fetch Meta Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":123,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_iter_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"state_store_iter_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fetch Meta Unhits","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock (Read)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":11},"height":null,"hideTimeOverride":false,"id":124,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric shows the real memory usage of uploader.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":125,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploading memory - {{job}} @ {{instance}}","metric":"","query":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_uploading_task_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploading task size - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_uploader_uploading_task_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_imm_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploader imm size - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_uploader_imm_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_imm_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance) - sum(state_store_uploader_uploading_task_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unflushed imm size - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_uploader_imm_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance) - sum(state_store_uploader_uploading_task_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance) - sum(state_store_uploader_imm_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"orphan imm size - {{job}} @ {{instance}}","metric":"","query":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance) - sum(state_store_uploader_imm_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_old_value_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"old value size - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_old_value_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader Memory Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of time spent on compacting shared buffer to remote storage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":126,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 Sync duration - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 Sync duration - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax Sync duration - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_sync_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg Sync duration - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance) (rate(state_store_sync_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_uploader_upload_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 upload task duration - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_uploader_upload_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_uploader_upload_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 upload task duration - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_uploader_upload_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_uploader_upload_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax upload task duration - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_uploader_upload_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Build and Sync Sstable Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":127,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.5, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write p50 - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(0.5, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.99, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write p99 - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(0.99, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write pmax - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Write Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":128,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_merge_imm_task_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"merge imm tasks - {{table_id}} @ {{instance}}","metric":"","query":"sum(irate(state_store_merge_imm_task_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_spill_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Uploader spill tasks - {{uploader_stage}} @ {{instance}}","metric":"","query":"sum(irate(state_store_spill_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_uploading_task_count{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploading task count - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_uploader_uploading_task_count{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_syncing_epoch_count{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"syncing epoch count - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_uploader_syncing_epoch_count{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader - Tasks Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":129,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_merge_imm_memory_sz{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Merging tasks memory size - {{table_id}} @ {{instance}}","metric":"","query":"sum(rate(state_store_merge_imm_memory_sz{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_spill_task_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Uploading tasks size - {{uploader_stage}} @ {{instance}}","metric":"","query":"sum(rate(state_store_spill_task_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader - Task Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":130,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write batch - {{table_id}} @ {{job}} @ {{instance}} ","metric":"","query":"sum(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"l0 - {{job}} @ {{instance}} ","metric":"","query":"sum(rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":131,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer avg - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":132,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_write_batch_tuple_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write_batch_kv_pair_count - {{table_id}} @ {{instance}}","metric":"","query":"sum(irate(state_store_write_batch_tuple_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Item Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":133,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_write_batch_size_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id) / sum(rate(state_store_write_batch_size_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"shared_buffer - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_write_batch_size_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id) / sum(rate(state_store_write_batch_size_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_shared_buffer_to_sstable_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance) / sum(rate(compactor_shared_buffer_to_sstable_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sync - {{job}} @ {{instance}}","metric":"","query":"sum(rate(compactor_shared_buffer_to_sstable_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance) / sum(rate(compactor_shared_buffer_to_sstable_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric shows the statistics of mem_table size on flush. By default only max (p100) is shown.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":134,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_write_batch_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, table_id, job, instance) (rate(state_store_write_batch_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance) (rate(state_store_write_batch_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, table_id, job, instance) (rate(state_store_write_batch_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Batch Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":135,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_mem_table_spill_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mem table spill table id - {{table_id}} @ {{instance}}","metric":"","query":"sum(irate(state_store_mem_table_spill_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Mem Table Spill Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":136,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Checkpoint Sync Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":137,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_event_handler_pending_event{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"sum(state_store_event_handler_pending_event{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Event handler pending event number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":138,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_event_handler_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, event_type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 {{event_type}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_event_handler_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, event_type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_event_handler_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, event_type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 {{event_type}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_event_handler_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, event_type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_event_handler_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, event_type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax {{event_type}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_event_handler_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, event_type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_uploader_wait_poll_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 finished_task_wait_poll {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_uploader_wait_poll_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_uploader_wait_poll_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 finished_task_wait_poll {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_uploader_wait_poll_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_uploader_wait_poll_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax finished_task_wait_poll {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_uploader_wait_poll_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Event handle latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock (Write)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":12},"height":null,"hideTimeOverride":false,"id":139,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of SSTables at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":140,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","query":"sum(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"SSTable Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size(KB) of SSTables at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":141,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","query":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"SSTable Size(KB)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The  of bytes that have been written by commit epoch per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":142,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_commit_write_throughput{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{table_id}}","metric":"","query":"sum(rate(storage_commit_write_throughput{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Commit Flush Bytes by Table","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have completed or failed","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":143,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_frequency{result!='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task_type}} - {{result}} - group-{{group}} @ {{compactor}}","metric":"","query":"sum(storage_level_compact_frequency{result!='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Failure Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have completed or failed","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":144,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_frequency{result='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task_type}} - {{result}} - group-{{group}} @ {{compactor}}","metric":"","query":"sum(storage_level_compact_frequency{result='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Success Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have been skipped.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":145,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_skip_compact_frequency{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (level, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{level}}-{{type}}","metric":"","query":"sum(rate(storage_skip_compact_frequency{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (level, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Skip Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Avg l0 select_level_count of the compact task, and categorize it according to different cg, levels and task types","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":146,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, group, type)(irate(storage_l0_compact_level_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, group, type)(irate(storage_l0_compact_level_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg cg{{group}}@{{type}}","metric":"","query":"sum by(le, group, type)(irate(storage_l0_compact_level_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, group, type)(irate(storage_l0_compact_level_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task L0 Select Level Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Avg file count of the compact task, and categorize it according to different cg, levels and task types","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":147,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, group, type)(irate(storage_compact_task_file_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, group, type)(irate(storage_compact_task_file_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg cg{{group}}@{{type}}","metric":"","query":"sum by(le, group, type)(irate(storage_compact_task_file_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, group, type)(irate(storage_compact_task_file_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task File Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The distribution of the compact task size triggered, including p90 and max. and categorize it according to different cg, levels and task types.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":148,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - cg{{group}}@{{type}}","metric":"","query":"histogram_quantile(0.9, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - cg{{group}}@{{type}}","metric":"","query":"histogram_quantile(1.0, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task Size Distribution","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that are running.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":149,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(storage_compact_task_pending_num{job=~\"$job\",instance=~\"$node\"}) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor_task_count - {{job}} @ {{instance}}","metric":"","query":"avg(storage_compact_task_pending_num{job=~\"$job\",instance=~\"$node\"}) by(job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(storage_compact_task_pending_parallelism{job=~\"$job\",instance=~\"$node\"}) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor_task_pending_parallelism - {{job}} @ {{instance}}","metric":"","query":"avg(storage_compact_task_pending_parallelism{job=~\"$job\",instance=~\"$node\"}) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compactor Running Task Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"compact-task: The total time have been spent on compaction.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":150,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task p50 - {{job}}","metric":"","query":"histogram_quantile(0.5, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task p90 - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task pmax - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range p90 - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range pmax - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get-table-id p90 - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get-table-id pmax - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io p90 - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io pmax - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(compute_refill_cache_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compute_apply_version_duration_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(compute_refill_cache_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le)(rate(compactor_compact_task_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(compactor_compact_task_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task avg","metric":"","query":"sum by(le)(rate(compactor_compact_task_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(compactor_compact_task_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le)(rate(state_store_compact_sst_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(state_store_compact_sst_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range avg","metric":"","query":"sum by(le)(rate(state_store_compact_sst_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(state_store_compact_sst_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"KBs read from next level during history compactions to next level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":151,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job) + sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}}","metric":"","query":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job) + sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}}","metric":"","query":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"flush - {{job}}","metric":"","query":"sum(rate(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_fast_compact_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fast compact - {{job}}","metric":"","query":"sum(rate(compactor_fast_compact_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of bytes that have been written by compaction.Flush refers to the process of compacting Memtables to SSTables at Level 0.Write refers to the process of compacting SSTables at one level to another level.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":152,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}}","metric":"","query":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"flush - {{job}}","metric":"","query":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Write Bytes(GiB)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Write amplification is the amount of bytes written to the remote storage by compaction for each one byte of flushed SSTable data. Write amplification is by definition higher than 1.0 because we write each piece of data to L0, and then write it again to an SSTable, and then compaction may read this piece of data and write it to a new SSTable, that's another write.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":153,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) / sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write amplification","metric":"","query":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) / sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Write Amplification","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of SSTables that is being compacted at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":154,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_level_compact_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","query":"storage_level_compact_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compacting SSTable Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"num of compact_task","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":155,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_level_compact_task_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task}}","metric":"","query":"storage_level_compact_task_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compacting Task Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":156,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from next level","metric":"","query":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from current level","metric":"","query":"sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} write to next level","metric":"","query":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"KBs Read/Write by Level","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":157,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_write_sstn{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} write to next level","metric":"","query":"sum(irate(storage_level_compact_write_sstn{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_read_sstn_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from next level","metric":"","query":"sum(irate(storage_level_compact_read_sstn_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_read_sstn_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from current level","metric":"","query":"sum(irate(storage_level_compact_read_sstn_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Count of SSTs Read/Write by level","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total bytes gotten from sstable_bloom_filter, for observing bloom_filter size","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":158,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_bloom_filter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_bloom_filter_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_meta - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_bloom_filter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_bloom_filter_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_file_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_file_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_file - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_file_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_file_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total bytes gotten from sstable_avg_key_size, for observing sstable_avg_key_size","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":159,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_key_size - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_value_size - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Item Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Avg count gotten from sstable_distinct_epoch_count, for observing sstable_distinct_epoch_count","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":160,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_distinct_epoch_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_distinct_epoch_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_epoch_count - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_distinct_epoch_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_distinct_epoch_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Stat","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total time of operations which read from remote storage when enable prefetch","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":161,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io p90 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Remote Read Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":88},"height":null,"hideTimeOverride":false,"id":162,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_iter_scan_key_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter keys flow - {{type}} @ {{instance}} ","metric":"","query":"sum(rate(compactor_iter_scan_key_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compactor Iter keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"bytes of Lsm tree needed to reach balance","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":88},"height":null,"hideTimeOverride":false,"id":163,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_compact_pending_bytes{job=~\"$job\",instance=~\"$node\"}) by (instance, group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact pending bytes - {{group}} @ {{instance}} ","metric":"","query":"sum(storage_compact_pending_bytes{job=~\"$job\",instance=~\"$node\"}) by (instance, group)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lsm Compact Pending Bytes","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"compression ratio of each level of the lsm tree","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":96},"height":null,"hideTimeOverride":false,"id":164,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_compact_level_compression_ratio{job=~\"$job\",instance=~\"$node\"}) by (instance, group, level, algorithm)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lsm compression ratio - cg{{group}} @ L{{level}} - {{algorithm}} {{instance}}","metric":"","query":"sum(storage_compact_level_compression_ratio{job=~\"$job\",instance=~\"$node\"}) by (instance, group, level, algorithm)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lsm Level Compression Ratio","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Compaction","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":13},"height":null,"hideTimeOverride":false,"id":165,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":166,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":167,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, type, job, instance)(rate(object_store_operation_latency_sum{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(object_store_operation_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, type, job, instance)(rate(object_store_operation_latency_sum{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(object_store_operation_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":168,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type!~'streaming_upload_write_bytes|streaming_read_read_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_operation_latency_count{type!~'streaming_upload_write_bytes|streaming_read_read_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type=~'upload|delete',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{media_type}}-write - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_operation_latency_count{type=~'upload|delete',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type=~'read|readv|list|metadata',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{media_type}}-read - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_operation_latency_count{type=~'read|readv|list|metadata',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":169,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":170,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Failure Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":171,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_request_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_request_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Retry Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"There are two types of operations: 1. GET, SELECT, and DELETE, they cost 0.0004 USD per 1000 requests. 2. PUT, COPY, POST, LIST, they cost 0.005 USD per 1000 requests.Reading from S3 across different regions impose extra cost. This metric assumes 0.01 USD per 1GB data transfer. Please checkout AWS's pricing model for more accurate calculation.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"$"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":172,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}) * 0.01 / 1000 / 1000 / 1000","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"(Cross Region) Data Transfer Cost","metric":"","query":"sum(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}) * 0.01 / 1000 / 1000 / 1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_operation_latency_count{type=~'read|streaming_read_start|streaming_read_init',job=~\"$job\",instance=~\"$node\"}) * 0.0004 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GET, SELECT, and all other Requests Cost","metric":"","query":"sum(object_store_operation_latency_count{type=~'read|streaming_read_start|streaming_read_init',job=~\"$job\",instance=~\"$node\"}) * 0.0004 / 1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_operation_latency_count{type=~'upload|streaming_upload|streaming_upload_start|s3_upload_part|streaming_upload_finish|list',job=~\"$job\",instance=~\"$node\"}) * 0.005 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"PUT, COPY, POST, LIST Requests Cost","metric":"","query":"sum(object_store_operation_latency_count{type=~'upload|streaming_upload|streaming_upload_start|s3_upload_part|streaming_upload_finish|list',job=~\"$job\",instance=~\"$node\"}) * 0.005 / 1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Estimated S3 Cost (Realtime)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric uses the total size of data in S3 at this second to derive the cost of storing data for a whole month. The price is 0.023 USD per GB. Please checkout AWS's pricing model for more accurate calculation.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"$"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":173,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance) * 0.023 / 1000 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Monthly Storage Cost","metric":"","query":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance) * 0.023 / 1000 / 1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Estimated S3 Cost (Monthly)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Object Storage","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":14},"height":null,"hideTimeOverride":false,"id":174,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":175,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_hybrid_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - hybrid - {{op}} @ {{instance}}","metric":"","query":"sum(rate(foyer_hybrid_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hybrid Cache Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":176,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{name}} - hybrid - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{name}} - hybrid - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{name}} - hybrid - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{name}} - hybrid - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hybrid Cache Op Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":177,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_hybrid_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) / (sum(rate(foyer_hybrid_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) + sum(rate(foyer_hybrid_op_total{op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - hybrid - hit ratio @ {{instance}}","metric":"","query":"sum(rate(foyer_hybrid_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) / (sum(rate(foyer_hybrid_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) + sum(rate(foyer_hybrid_op_total{op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hybrid Cache Hit Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":178,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_memory_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - memory - {{op}} @ {{instance}}","metric":"","query":"sum(rate(foyer_memory_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Cache Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":179,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(foyer_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (name, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - memory - size @ {{instance}}","metric":"","query":"sum(foyer_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (name, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Cache Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":180,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_memory_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) / (sum(rate(foyer_memory_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) + sum(rate(foyer_memory_op_total{op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - memory - hit ratio @ {{instance}}","metric":"","query":"sum(rate(foyer_memory_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) / (sum(rate(foyer_memory_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) + sum(rate(foyer_memory_op_total{op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Cache Hit Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":181,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"sum(rate(foyer_storage_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":182,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_inner_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"sum(rate(foyer_storage_inner_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache Inner Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":183,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache Op Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":184,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache Inner Op Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":185,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) / (sum(rate(foyer_storage_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) + sum(rate(foyer_storage_op_total{op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - storage - hit ratio @ {{instance}}","metric":"","query":"sum(rate(foyer_storage_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) / (sum(rate(foyer_storage_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) + sum(rate(foyer_storage_op_total{op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache Hit Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":186,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(foyer_storage_region{job=~\"$job\",instance=~\"$node\"}) by (name, type, instance) * on(name, instance) group_left() avg(foyer_storage_region_size_bytes{job=~\"$job\",instance=~\"$node\"}) by (name, type, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - {{type}} region - size @ {{instance}}","metric":"","query":"sum(foyer_storage_region{job=~\"$job\",instance=~\"$node\"}) by (name, type, instance) * on(name, instance) group_left() avg(foyer_storage_region_size_bytes{job=~\"$job\",instance=~\"$node\"}) by (name, type, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Region Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":187,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_disk_io_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - disk - {{op}} @ {{instance}}","metric":"","query":"sum(rate(foyer_storage_disk_io_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Disk Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":188,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{name}} - disk - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{name}} - disk - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{name}} - disk - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{name}} - disk - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Disk Op Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":189,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_disk_io_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - disk - {{op}} @ {{instance}}","metric":"","query":"sum(rate(foyer_storage_disk_io_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Disk Op Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":190,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (type, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} file cache refill - {{op}} @ {{instance}}","metric":"","query":"sum(rate(refill_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (type, op, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=~\"meta|data\",op!~\"filtered|ignored\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (type, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} file cache refill - {{op}} @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=~\"meta|data\",op!~\"filtered|ignored\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (type, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":191,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} file cache - {{op}} @ {{instance}}","metric":"","query":"sum(rate(refill_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Data Refill Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":192,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{foyer}} cache refill - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{foyer}} cache refill - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{foyer}} cache refill - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{foyer}} cache refill - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":193,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(refill_queue_total) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"refill queue length @ {{instance}}","metric":"","query":"sum(refill_queue_total) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill Queue Length","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":194,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"parent_meta\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"parent meta lookup {{op}} @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=\"parent_meta\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inheritance - Parent Meta Lookup Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":195,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"parent_meta\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / (sum(rate(refill_total{type=\"parent_meta\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) + sum(rate(refill_total{type=\"parent_meta\",op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"parent meta lookup hit ratio @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=\"parent_meta\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / (sum(rate(refill_total{type=\"parent_meta\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) + sum(rate(refill_total{type=\"parent_meta\",op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)) >= 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inheritance - Parent Meta Lookup Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":196,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"unit_inheritance\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unit inheritance {{op}} @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=\"unit_inheritance\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inheritance - Unit inheritance Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":88},"height":null,"hideTimeOverride":false,"id":197,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"unit_inheritance\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / (sum(rate(refill_total{type=\"unit_inheritance\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) + sum(rate(refill_total{type=\"unit_inheritance\",op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unit inheritance ratio @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=\"unit_inheritance\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / (sum(rate(refill_total{type=\"unit_inheritance\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) + sum(rate(refill_total{type=\"unit_inheritance\",op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)) >= 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inheritance - Unit inheritance Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":88},"height":null,"hideTimeOverride":false,"id":198,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"block\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block refill {{op}} @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=\"block\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Block Refill Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":96},"height":null,"hideTimeOverride":false,"id":199,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"block\",op=\"success\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / sum(rate(refill_total{type=\"block\",op=\"unfiltered\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block refill ratio @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=\"block\",op=\"success\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / sum(rate(refill_total{type=\"block\",op=\"unfiltered\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) >= 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Block Refill Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Item numbers of the recent filter.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":96},"height":null,"hideTimeOverride":false,"id":200,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(recent_filter_items{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"items @ {{instance}}","metric":"","query":"sum(recent_filter_items{job=~\"$job\",instance=~\"$node\"}) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Recent Filter Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":104},"height":null,"hideTimeOverride":false,"id":201,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(recent_filter_ops{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recent filter {{op}} @ {{instance}}","metric":"","query":"sum(rate(recent_filter_ops{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Recent Filter Ops","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock Tiered Cache","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":15},"height":null,"hideTimeOverride":false,"id":202,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":203,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time p50 - {{method}} @ {{lock_type}} @ {{lock_name}}","metric":"","query":"histogram_quantile(0.5, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method, lock_name, lock_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time p99 - {{method}} @ {{lock_type}} @ {{lock_name}}","metric":"","query":"histogram_quantile(0.99, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method, lock_name, lock_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time pmax - {{method}} @ {{lock_type}} @ {{lock_name}}","metric":"","query":"histogram_quantile(1.0, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method, lock_name, lock_type))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lock Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":204,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time p50 - {{method}}","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time p99 - {{method}}","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time pmax - {{method}}","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Real Process Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":205,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version size","metric":"","query":"storage_version_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":206,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"current version id","metric":"","query":"storage_current_version_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"checkpoint version id","metric":"","query":"storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min pinned version id","metric":"","query":"storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_safepoint_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min safepoint version id","metric":"","query":"storage_min_safepoint_version_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Id","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":207,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"max committed epoch","metric":"","query":"storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_safe_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"safe epoch","metric":"","query":"storage_safe_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min pinned epoch","metric":"","query":"storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Epoch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":208,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_key_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","query":"storage_version_stats{metric='total_key_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_value_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","query":"storage_version_stats{metric='total_value_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Table Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":209,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_materialized_view_stats{metric='materialized_view_total_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{metric}}, mv id - {{table_id}} ","metric":"","query":"storage_materialized_view_stats{metric='materialized_view_total_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":210,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_key_count',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","query":"storage_version_stats{metric='total_key_count',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Table KV Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"\nObjects are classified into 3 groups:\n- not referenced by versions: these object are being deleted from object store.\n- referenced by non-current versions: these objects are stale (not in the latest version), but those old versions may still be in use (e.g. long-running pinning). Thus those objects cannot be deleted at the moment.\n- referenced by current version: these objects are in the latest version.\n\nAdditionally, a metric on all objects (including dangling ones) is updated with low-frequency. The metric is updated right before full GC. So subsequent full GC may reduce the actual value significantly, without updating the metric.\n                    ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":211,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","query":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","query":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","query":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_total_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all objects (including dangling ones)","metric":"","query":"storage_total_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Refer to `Object Total Number` panel for classification of objects.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":212,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","query":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","query":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","query":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_total_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all objects, including dangling ones","metric":"","query":"storage_total_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"total number of hummock version delta log","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":213,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_delta_log_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"delta log total number","metric":"","query":"storage_delta_log_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Delta Log Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"hummock version checkpoint latency","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":214,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(storage_version_checkpoint_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(storage_version_checkpoint_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_avg","metric":"","query":"rate(storage_version_checkpoint_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(storage_version_checkpoint_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Checkpoint Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"When certain per compaction group threshold is exceeded (e.g. number of level 0 sub-level in LSMtree), write op to that compaction group is stopped temporarily. Check log for detail reason of write stop.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":215,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_write_stop_compaction_groups{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compaction_group_{{compaction_group_id}}","metric":"","query":"storage_write_stop_compaction_groups{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Stop Compaction Groups","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"total number of attempts to trigger full GC","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":216,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_full_gc_trigger_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"full_gc_trigger_count","metric":"","query":"storage_full_gc_trigger_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Full GC Trigger Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"the object id watermark used in last full GC","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":217,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_full_gc_last_object_id_watermark{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"full_gc_last_object_id_watermark","metric":"","query":"storage_full_gc_last_object_id_watermark{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Full GC Last Watermark","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":218,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Event Loop Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The times of move_state_table occurs","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":219,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_move_state_table_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"move table cg{{group}}","metric":"","query":"sum(storage_move_state_table_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}) by (group)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Move State Table Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of state_tables in each CG","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":220,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_state_table_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"state table cg{{group}}","metric":"","query":"sum(irate(storage_state_table_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"State Table Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of branched_sst in each CG","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":221,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_branched_sst_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"branched sst cg{{group}}","metric":"","query":"sum(irate(storage_branched_sst_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Branched SST Count","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":222,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total backup job count since the Meta node starts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":223,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"backup_job_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"job count","metric":"","query":"backup_job_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Job Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Latency of backup jobs since the Meta node starts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":224,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time p50 - {{state}}","metric":"","query":"histogram_quantile(0.5, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time p99 - {{state}}","metric":"","query":"histogram_quantile(0.99, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time pmax - {{state}}","metric":"","query":"histogram_quantile(1.0, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Job Process Time","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Backup Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":17},"height":null,"hideTimeOverride":false,"id":225,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":226,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":227,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Drop latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":228,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"GetCatalog latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Catalog Service","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":18},"height":null,"hideTimeOverride":false,"id":229,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":230,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"AddWorkerNode latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":231,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"ListAllNodes latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Cluster Service","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":19},"height":null,"hideTimeOverride":false,"id":232,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":233,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CreateMaterializedView latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":234,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"DropMaterializedView latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":235,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Flush latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Stream Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":20},"height":null,"hideTimeOverride":false,"id":236,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":237,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UnpinVersionBefore latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":238,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UnpinSnapshotBefore latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":239,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"ReportCompactionTasks latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":240,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"GetNewSstIds latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Hummock Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":21},"height":null,"hideTimeOverride":false,"id":241,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":242,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_report_compaction_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_counts - {{instance}}","metric":"","query":"sum(irate(state_store_report_compaction_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"compaction_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":243,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_version_before_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_version_before_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_avg","metric":"","query":"sum(irate(state_store_unpin_version_before_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_version_before_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"version_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":244,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latencyp90 - {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_pin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_pin_snapshot_latency_count[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_avg","metric":"","query":"sum(irate(state_store_pin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_pin_snapshot_latency_count[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_snapshot_latency_count[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_avg","metric":"","query":"sum(irate(state_store_unpin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_snapshot_latency_count[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_unpin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(irate(state_store_unpin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"snapshot_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":245,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_pin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_counts - {{instance}}","metric":"","query":"sum(irate(state_store_pin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_counts - {{instance}}","metric":"","query":"sum(irate(state_store_unpin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"snapshot_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":246,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_get_new_sst_ids_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_get_new_sst_ids_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_avg","metric":"","query":"sum(irate(state_store_get_new_sst_ids_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_get_new_sst_ids_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"table_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":247,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_get_new_sst_ids_latency_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_counts - {{instance}}","metric":"","query":"sum(irate(state_store_get_new_sst_ids_latency_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"table_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":248,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_report_compaction_task_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_report_compaction_task_latency_count[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_avg","metric":"","query":"sum(irate(state_store_report_compaction_task_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_report_compaction_task_latency_count[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"compaction_latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC: Hummock Meta Client","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":22},"height":null,"hideTimeOverride":false,"id":249,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of active sessions","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":250,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Active Sessions","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":251,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Per Second (Local Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":252,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Per Second (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":253,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of running query in distributed execution mode","metric":"","query":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Running Queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":254,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of rejected query in distributed execution mode","metric":"","query":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Rejected queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":255,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of completed query in distributed execution mode","metric":"","query":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Completed Queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":256,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":257,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency (Local Query Mode)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Frontend","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":23},"height":null,"hideTimeOverride":false,"id":258,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":259,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(lru_runtime_loop_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"rate(lru_runtime_loop_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager loop count per sec","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":260,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_eviction_policy{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"lru_eviction_policy{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager eviction policy","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":261,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_latest_sequence{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"lru_latest_sequence{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_watermark_sequence{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"lru_watermark_sequence{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager sequence","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":262,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jemalloc_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The allocated memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":263,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_active_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jemalloc_active_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The active memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":264,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_resident_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jemalloc_resident_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The resident memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":265,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_metadata_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jemalloc_metadata_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The metadata memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":266,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jvm_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jvm_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The allocated memory of jvm","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":267,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jvm_active_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jvm_active_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The active memory of jvm","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":268,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_current_watermark_time_ms{job=~\"$job\",instance=~\"$node\"} - on() group_right() lru_evicted_watermark_time_ms{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table {{table_id}} actor {{actor_id}} desc: {{desc}}","metric":"","query":"lru_current_watermark_time_ms{job=~\"$job\",instance=~\"$node\"} - on() group_right() lru_evicted_watermark_time_ms{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager diff between current watermark and evicted watermark time (ms) for actors","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Memory manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":269,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":270,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(source_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_type}} @ {{source_id}}","metric":"","query":"rate(source_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Source Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":271,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(connector_sink_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector_type}} @ {{sink_id}}","metric":"","query":"rate(connector_sink_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Sink Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Connector Node","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":25},"height":null,"hideTimeOverride":false,"id":272,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":273,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 @ {{connector}} {{sink_id}}","metric":"","query":"histogram_quantile(0.5, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 @ {{connector}} {{sink_id}}","metric":"","query":"histogram_quantile(0.99, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax @ {{connector}} {{sink_id}}","metric":"","query":"histogram_quantile(1.0, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, connector, sink_id)(rate(sink_commit_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(sink_commit_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{connector}} @ {{sink_id}}","metric":"","query":"sum by(le, connector, sink_id)(rate(sink_commit_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(sink_commit_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Commit Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":274,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"log_store_latest_write_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"latest write epoch @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","query":"log_store_latest_write_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"latest read epoch @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","query":"log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"kv_log_store_buffer_unconsumed_min_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Kv log store uncomsuned min epoch @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","query":"kv_log_store_buffer_unconsumed_min_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Read/Write Epoch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":275,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(max(log_store_latest_write_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)- max(log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)) / (2^16) / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Consume lag @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","query":"(max(log_store_latest_write_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)- max(log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)) / (2^16) / 1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Lag","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":276,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(log_store_reader_wait_new_future_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id, executor_id) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Backpressure @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","query":"avg(rate(log_store_reader_wait_new_future_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id, executor_id) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Backpressure Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":277,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_min((max(log_store_first_write_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)- max(log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)) / (2^16) / 1000, 0)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Consume persistent log lag @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","query":"clamp_min((max(log_store_first_write_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)- max(log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)) / (2^16) / 1000, 0)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Consume Persistent Log Lag","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":278,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_read_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector}} {{sink_id}}","metric":"","query":"sum(rate(log_store_read_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Consume Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":279,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_read_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, connector, sink_id, executor_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector}} {{sink_id}} @ {{executor_id}} {{instance}}","metric":"","query":"sum(rate(log_store_read_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, connector, sink_id, executor_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Log Store Consume Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":280,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_write_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector}} {{sink_id}}","metric":"","query":"sum(rate(log_store_write_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Write Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":281,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_write_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, connector, sink_id, executor_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector}} {{sink_id}} @ {{executor_id}} {{instance}}","metric":"","query":"sum(rate(log_store_write_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, connector, sink_id, executor_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Log Store Write Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":282,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_read_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","query":"sum(rate(kv_log_store_storage_read_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Read Storage Row Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":283,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_read_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","query":"sum(rate(kv_log_store_storage_read_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Read Storage Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":284,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_write_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","query":"sum(rate(kv_log_store_storage_write_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Write Storage Row Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":285,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_write_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","query":"sum(rate(kv_log_store_storage_write_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Write Storage Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":286,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"kv_log_store_buffer_unconsumed_item_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Unconsumed item count @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","query":"kv_log_store_buffer_unconsumed_item_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"kv_log_store_buffer_unconsumed_row_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Unconsumed row count @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","query":"kv_log_store_buffer_unconsumed_row_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"kv_log_store_buffer_unconsumed_epoch_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Unconsumed epoch count @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","query":"kv_log_store_buffer_unconsumed_epoch_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Buffer State","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":287,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_rewind_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","query":"sum(rate(kv_log_store_rewind_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Rewind Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":288,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(kv_log_store_rewind_delay_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, executor_id, connector, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","query":"histogram_quantile(1.0, sum(rate(kv_log_store_rewind_delay_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, executor_id, connector, sink_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Rewind delay (second)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total size of chunks buffered in a barrier","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":289,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_sink_chunk_buffer_size{job=~\"$job\",instance=~\"$node\"}) by (sink_id, actor_id) * on(actor_id) group_left(sink_name) sink_info{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}} - actor {{actor_id}}","metric":"","query":"sum(stream_sink_chunk_buffer_size{job=~\"$job\",instance=~\"$node\"}) by (sink_id, actor_id) * on(actor_id) group_left(sink_name) sink_info{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Chunk Buffer Size","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Sink Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":26},"height":null,"hideTimeOverride":false,"id":290,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Kafka high watermark by source and partition and source latest message by partition, source and actor","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":291,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_kafka_high_watermark{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"high watermark: source={{source_id}} partition={{partition}}","metric":"","query":"source_kafka_high_watermark{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_latest_message_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"latest msg: source={{source_id}} partition={{partition}} actor_id={{actor_id}}","metric":"","query":"source_latest_message_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kafka high watermark and source latest message","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Current number of messages in producer queues","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":292,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_msg_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Count in Producer Queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Current total size of messages in producer queues","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":293,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_msg_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_msg_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Size in Producer Queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of messages transmitted (produced) to Kafka brokers","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":294,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_tx_msgs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_tx_msgs{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Produced Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of messages consumed, not including ignored messages (due to offset, etc), from Kafka brokers.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":295,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_rx_msgs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_rx_msgs{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Received Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages awaiting transmission to broker","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":296,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_outbuf_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_outbuf_msg_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Count Pending to Transmit (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages in-flight to broker awaiting response","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":297,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_waitresp_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_waitresp_msg_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inflight Message Count (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of transmission errors","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":298,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_tx_errs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_tx_errs{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Error Count When Transmitting (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of receive errors","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":299,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rx_errs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_rx_errs{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Error Count When Receiving (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of requests timed out","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":300,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_req_timeouts{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_req_timeouts{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Timeout Request Count (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Broker latency / round-trip time in milli seconds","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":301,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_avg{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_avg{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p75{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_p75{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p90{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_p90{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_p99{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"RTT (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Broker throttling time in milliseconds","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":302,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_avg{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_avg{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p75{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_p75{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p90{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_p90{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_p99{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Throttle Time (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Age of metadata from broker for this topic (milliseconds)","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":303,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_metadata_age{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}","metric":"","query":"rdkafka_topic_metadata_age{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Topic Metadata_age Age","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Batch sizes in bytes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":304,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_avg{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_avg{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p75{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_p75{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p90{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_p90{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_p99{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p99_99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_p99_99{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_out_of_range{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_out_of_range{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Batch message counts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_avg{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_avg{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p75{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_p75{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p90{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_p90{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_p99{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p99_99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_p99_99{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_out_of_range{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_out_of_range{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Topic Batch Messages","transformations":[],"transparent":false,"type":"timeseries"}],"timeFrom":null,"timeShift":null,"title":"Topic Batch Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages ready to be produced in transmit queue","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":305,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_xmit_msgq_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","query":"rdkafka_topic_partition_xmit_msgq_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message to be Transmitted","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of pre-fetched messages in fetch queue","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":306,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_fetchq_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","query":"rdkafka_topic_partition_fetchq_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message in pre fetch queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Next offset to fetch","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":307,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_next_offset{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","query":"rdkafka_topic_partition_next_offset{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Next offset to fetch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Last committed offset","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":308,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_committed_offset{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","query":"rdkafka_topic_partition_committed_offset{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Committed Offset","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Kafka Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":27},"height":null,"hideTimeOverride":false,"id":309,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":310,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} read @ {{instance}}","metric":"","query":"sum(rate(connection_read_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} write @ {{instance}}","metric":"","query":"sum(rate(connection_write_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Network throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":311,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} read @ {{instance}}","metric":"","query":"sum(rate(connection_read_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} write @ {{instance}}","metric":"","query":"sum(rate(connection_write_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"S3 throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":312,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} read @ {{instance}}","metric":"","query":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} write @ {{instance}}","metric":"","query":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} total read @ {{instance}}","metric":"","query":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} total write @ {{instance}}","metric":"","query":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"gRPC throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":313,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_io_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 {{op_type}} err[{{error_kind}}] @ {{instance}}","metric":"","query":"sum(irate(connection_io_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_io_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} grpc {{op_type}} err[{{error_kind}}] @ {{instance}}","metric":"","query":"sum(rate(connection_io_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_io_err_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} total {{op_type}} err[{{error_kind}}] @ {{instance}}","metric":"","query":"sum(rate(connection_io_err_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"IO error rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":314,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(connection_count{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 @ {{instance}}","metric":"","query":"sum(connection_count{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(connection_count{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}) by (job, instance, connection_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} @ {{instance}}","metric":"","query":"sum(connection_count{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}) by (job, instance, connection_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Existing connection count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":315,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_create_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 @ {{instance}}","metric":"","query":"sum(irate(connection_create_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_create_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} @ {{instance}}","metric":"","query":"sum(irate(connection_create_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create new connection rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":316,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 @ {{instance}}","metric":"","query":"sum(irate(connection_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} @ {{instance}}","metric":"","query":"sum(irate(connection_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create new connection err rate","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Network connection","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":28},"height":null,"hideTimeOverride":false,"id":317,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"iceberg write qps","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":318,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"iceberg_write_qps{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} @ {{sink_id}}","metric":"","query":"iceberg_write_qps{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Qps Of Iceberg Writer","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":319,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 @ {{sink_id}}","metric":"","query":"histogram_quantile(0.5, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 @ {{sink_id}}","metric":"","query":"histogram_quantile(0.99, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax @ {{sink_id}}","metric":"","query":"histogram_quantile(1.0, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, sink_id)(rate(iceberg_write_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(iceberg_write_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg @ {{sink_id}}","metric":"","query":"sum by(le, sink_id)(rate(iceberg_write_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(iceberg_write_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Latency Of Iceberg Writer","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":320,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"iceberg_rolling_unfushed_data_file{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} @ {{sink_id}}","metric":"","query":"iceberg_rolling_unfushed_data_file{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iceberg rolling unfushed data file","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":321,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"iceberg_position_delete_cache_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} @ {{sink_id}}","metric":"","query":"iceberg_position_delete_cache_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iceberg position delete cache num","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":322,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"iceberg_partition_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} @ {{sink_id}}","metric":"","query":"iceberg_partition_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iceberg partition num","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Iceberg Sink Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":29},"height":null,"hideTimeOverride":false,"id":323,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":324,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_success_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_success_count - {{instance}}","metric":"","query":"sum(rate(udf_success_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_failure_count - {{instance}}","metric":"","query":"sum(rate(udf_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_retry_count - {{instance}}","metric":"","query":"sum(rate(udf_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_success_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_success_count - {{link}} {{name}} {{fragment_id}}","metric":"","query":"sum(rate(udf_success_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_failure_count - {{link}} {{name}} {{fragment_id}}","metric":"","query":"sum(rate(udf_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_retry_count - {{instance}}","metric":"","query":"sum(rate(udf_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Calls Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":325,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(udf_input_chunk_rows_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / sum(irate(udf_input_chunk_rows_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_input_chunk_rows_avg - {{link}} {{name}} {{fragment_id}}","metric":"","query":"sum(irate(udf_input_chunk_rows_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / sum(irate(udf_input_chunk_rows_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Input Chunk Rows","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":326,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.50, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.50, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(udf_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / sum(irate(udf_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_avg - {{instance}}","metric":"","query":"sum(irate(udf_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / sum(irate(udf_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, link, name, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_p99_by_name - {{link}} {{name}} {{fragment_id}}","metric":"","query":"histogram_quantile(0.99, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, link, name, fragment_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(udf_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / sum(irate(udf_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_avg_by_name - {{link}} {{name}} {{fragment_id}}","metric":"","query":"sum(irate(udf_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / sum(irate(udf_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":327,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_input_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_throughput_rows - {{instance}}","metric":"","query":"sum(rate(udf_input_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_input_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_throughput_rows - {{link}} {{name}} {{fragment_id}}","metric":"","query":"sum(rate(udf_input_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Throughput (rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":328,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_throughput_bytes - {{instance}}","metric":"","query":"sum(rate(udf_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_throughput_bytes - {{link}} {{name}} {{fragment_id}}","metric":"","query":"sum(rate(udf_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / (1024*1024)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Throughput (bytes)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Currently only embedded JS UDF supports this. Others will always show 0.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":329,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(udf_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_memory_usage - {{instance}}","metric":"","query":"sum(udf_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(udf_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_memory_usage - {{name}} {{fragment_id}}","metric":"","query":"sum(udf_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Memory Usage (bytes)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"User Defined Function","transformations":[],"transparent":false,"type":"row"}],"refresh":"","rows":[],"schemaVersion":12,"sharedCrosshair":true,"style":"dark","tags":["risingwave"],"templating":{"list":[{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, instance)","description":"Reporting instance of the metric","hide":0,"includeAll":true,"label":"Node","multi":true,"name":"node","options":[],"query":{"query":"label_values(process_cpu_seconds_total, instance)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, job)","description":"Reporting job of the metric","hide":0,"includeAll":true,"label":"Job","multi":true,"name":"job","options":[],"query":{"query":"label_values(process_cpu_seconds_total, job)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(table_info, table_id)","description":"Reporting table id of the metric","hide":0,"includeAll":true,"label":"Table","multi":true,"name":"table","options":[],"query":{"query":"label_values(table_info, table_id)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"}]},"time":{"from":"now-30m","to":"now"},"timepicker":{"hidden":false,"nowDelay":null,"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone":"browser","title":"risingwave_dev_dashboard","uid":"Ecy3uV1nz","version":0}
+{"__inputs":[],"annotations":{"list":[]},"description":"RisingWave Dev Dashboard","editable":true,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"panels":[{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":1,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Information about actors","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]}},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":2,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true},"repeat":null,"repeatDirection":null,"span":6,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"group(actor_info{job=~\"$job\",instance=~\"$node\"}) by (actor_id, fragment_id, compute_node)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"group(actor_info{job=~\"$job\",instance=~\"$node\"}) by (actor_id, fragment_id, compute_node)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true},"indexByName":{"actor_id":0,"compute_node":2,"fragment_id":1}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Information about state tables. Column `materialized_view_id` is the id of the materialized view that this state table belongs to.","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]}},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":3,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true},"repeat":null,"repeatDirection":null,"span":6,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name, table_type, materialized_view_id, fragment_id, compaction_group_id)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name, table_type, materialized_view_id, fragment_id, compaction_group_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"State Table Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true},"indexByName":{"compaction_group_id":5,"fragment_id":4,"materialized_view_id":3,"table_id":0,"table_name":1,"table_type":2}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Actor count per compute node","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]}},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":4,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true},"repeat":null,"repeatDirection":null,"span":6,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"count(actor_info{job=~\"$job\",instance=~\"$node\"}) by (compute_node)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"count(actor_info{job=~\"$job\",instance=~\"$node\"}) by (compute_node)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Count (Group By Compute Node)","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true},"indexByName":{"compaction_group_id":5,"fragment_id":4,"materialized_view_id":3,"table_id":0,"table_name":1,"table_type":2}}}],"transparent":false,"type":"table"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Actor/Table Id Info","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":1},"height":null,"hideTimeOverride":false,"id":5,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of each type of RisingWave components alive.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":6,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_type}}","metric":"","query":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The memory usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":7,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Memory","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Memory usage relative to k8s resource limit of container. Only works in K8s environment","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":8,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(avg by(namespace, pod) (container_memory_working_set_bytes{namespace=~\"$namespace\",pod=~\"$pod\",container=~\"$component\"})) / (  sum by(namespace, pod) (kube_pod_container_resource_limits{namespace=~\"$namespace\", pod=~\"$pod\", container=\"$component\", resource=\"memory\", unit=\"byte\"}))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg memory usage @ {{job}} @ {{instance}}","metric":"","query":"(avg by(namespace, pod) (container_memory_working_set_bytes{namespace=~\"$namespace\",pod=~\"$pod\",container=~\"$component\"})) / (  sum by(namespace, pod) (kube_pod_container_resource_limits{namespace=~\"$namespace\", pod=~\"$pod\", container=\"$component\", resource=\"memory\", unit=\"byte\"}))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Memory relative","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The CPU usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":9,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cpu usage (total) - {{job}} @ {{instance}}","metric":"","query":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (job, instance) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cpu usage (avg per core) - {{job}} @ {{instance}}","metric":"","query":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (job, instance) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"CPU usage relative to k8s resource limit of container. Only works in K8s environment","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":10,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\",container=~\"$component\",pod=~\"$pod\"}[$__rate_interval])) by (namespace, pod)) / (sum(kube_pod_container_resource_limits{namespace=~\"$namespace\",pod=~\"$pod\",container=~\"$component\", resource=\"cpu\"}) by (namespace, pod))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cpu usage @ {{job}} @ {{instance}}","metric":"","query":"(sum(rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\",container=~\"$component\",pod=~\"$pod\"}[$__rate_interval])) by (namespace, pod)) / (sum(kube_pod_container_resource_limits{namespace=~\"$namespace\",pod=~\"$pod\",container=~\"$component\", resource=\"cpu\"}) by (namespace, pod))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU relative","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"RW cluster can configure multiple meta nodes to achieve high availability. One is the leader and the rest are the followers.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":11,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(meta_num{job=~\"$job\",instance=~\"$node\"}) by (worker_addr,role)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_addr}} @ {{role}}","metric":"","query":"sum(meta_num{job=~\"$job\",instance=~\"$node\"}) by (worker_addr,role)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Meta Cluster","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Cluster Node","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":2},"height":null,"hideTimeOverride":false,"id":12,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The rate of successful recovery attempts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":13,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Recovery Successful Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of failed reocovery attempts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":14,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Failed recovery attempts","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Time spent in a successful recovery attempt","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":15,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency pmax - {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by (le) (rate(recovery_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by (le) (rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency avg","metric":"","query":"sum by (le) (rate(recovery_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by (le) (rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Recovery latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Recovery","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":3},"height":null,"hideTimeOverride":false,"id":16,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of barriers that have been ingested but not completely processed. This metric reflects the current level of congestion within the system.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":17,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all_barrier","metric":"","query":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"in_flight_barrier_nums{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"in_flight_barrier","metric":"","query":"in_flight_barrier_nums{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The time that the data between two consecutive barriers gets fully processed, i.e. the computation results are made durable into materialized views or sink to external systems. This metric shows to users the freshness of materialized views.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":18,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_avg","metric":"","query":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The duration from the last committed barrier's epoch time to the current time. This metric reflects the data freshness of the system. During this time, no new data has been committed.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":19,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"timestamp(last_committed_barrier_time{job=~\"$job\",instance=~\"$node\"}) - last_committed_barrier_time{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_pending_time","metric":"","query":"timestamp(last_committed_barrier_time{job=~\"$job\",instance=~\"$node\"}) - last_committed_barrier_time{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier pending time (secs)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":20,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","query":"sum(rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Each query is executed in parallel with a user-defined parallelism. This figure shows the throughput of each parallelism. The throughput of all the parallelism added up is equal to Source Throughput(rows).","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":21,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(source_partition_input_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor={{actor_id}} source={{source_id}} partition={{partition}} fragment_id={{fragment_id}}","metric":"","query":"rate(source_partition_input_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s) Per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of bytes read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":22,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum by (source_id, source_name, fragment_id)(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","query":"(sum by (source_id, source_name, fragment_id)(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Each query is executed in parallel with a user-defined parallelism. This figure shows the throughput of each parallelism. The throughput of all the parallelism added up is equal to Source Throughput(MB/s).","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":23,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor={{actor_id}} source={{source_id}} partition={{partition}} fragment_id={{fragment_id}}","metric":"","query":"(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))/(1000*1000)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s) Per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":24,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_source_backfill_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","query":"sum(rate(stream_source_backfill_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Backfill Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Monitor each source upstream, 0 means the upstream is not normal, 1 means the source is ready.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":25,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_status_is_up{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source_id={{source_id}}, source_name={{source_name}} @ {{instance}}","metric":"","query":"source_status_is_up{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Upstream Status","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Source Split Change Events frequency by source_id and actor_id","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":26,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_source_split_change_event_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_name}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_source_split_change_event_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Split Change Events frequency(events/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Kafka Consumer Lag Size by source_id, partition and actor_id","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":27,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_min(source_kafka_high_watermark{job=~\"$job\",instance=~\"$node\"} - on(source_id, partition) group_right() source_latest_message_id{job=~\"$job\",instance=~\"$node\"}, 0)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_id}} partition={{partition}} actor_id={{actor_id}}","metric":"","query":"clamp_min(source_kafka_high_watermark{job=~\"$job\",instance=~\"$node\"} - on(source_id, partition) group_right() source_latest_message_id{job=~\"$job\",instance=~\"$node\"}, 0)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kafka Consumer Lag Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of rows streamed into each sink per second. For sinks with 'sink_decouple = true', please refer to the 'Sink Metrics' section","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":28,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id) * on(sink_id) group_left(sink_name) group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}}","metric":"","query":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id) * on(sink_id) group_left(sink_name) group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (sink_id, sink_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Throughput(rows/s) *","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of rows streamed into each sink per second. For sinks with 'sink_decouple = true', please refer to the 'Sink Metrics' section","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":29,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id, actor_id) * on(actor_id) group_left(sink_name) sink_info{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}} - actor {{actor_id}}","metric":"","query":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id, actor_id) * on(actor_id) group_left(sink_name) sink_info{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Throughput(rows/s) per Partition *","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized view per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":30,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id) * on(table_id) group_left(table_name) group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mview {{table_id}} {{table_name}}","metric":"","query":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id) * on(table_id) group_left(table_name) group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized view per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":31,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) * on(fragment_id, table_id) group_left(table_name) table_info{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mview {{table_id}} {{table_name}} - actor {{actor_id}} fragment_id {{fragment_id}}","metric":"","query":"rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) * on(fragment_id, table_id) group_left(table_name) table_info{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s) per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been read from the backfill snapshot","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":32,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_snapshot_read_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_backfill_snapshot_read_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Snapshot Read Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been output from the backfill upstream","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":33,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_upstream_output_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_backfill_upstream_output_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Upstream Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The duration between the time point when the scheduled barrier needs to be sent and the time point when the barrier gets actually sent to all the compute nodes. Developers can thus detect any internal congestion.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":34,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_send_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_send_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_avg","metric":"","query":"rate(meta_barrier_send_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_send_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Send Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":35,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"max(sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_avg","metric":"","query":"max(sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier In-Flight Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":36,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p999 - {{instance}}","metric":"","query":"histogram_quantile(0.999, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_pmax - {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_avg - {{instance}}","metric":"","query":"sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Sync Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":37,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_wait_commit_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_wait_commit_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_avg","metric":"","query":"rate(meta_barrier_wait_commit_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_wait_commit_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Wait Commit Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of actors that have processed the earliest in-flight barriers per second. This metric helps users to detect potential congestion or stuck in the system.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":38,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_barrier_manager_progress{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"rate(stream_barrier_manager_progress{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Earliest In-Flight Barrier Progress","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":4},"height":null,"hideTimeOverride":false,"id":39,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been read from the cdc backfill snapshot","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":40,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_cdc_backfill_snapshot_read_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_cdc_backfill_snapshot_read_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CDC Backfill Snapshot Read Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been output from the cdc backfill upstream","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":41,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_cdc_backfill_upstream_output_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_cdc_backfill_upstream_output_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CDC Backfill Upstream Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":42,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lag p50 - {{table_name}}","metric":"","query":"histogram_quantile(0.5, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lag p99 - {{table_name}}","metric":"","query":"histogram_quantile(0.99, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lag pmax - {{table_name}}","metric":"","query":"histogram_quantile(1.0, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CDC Consume Lag Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":43,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(cdc_source_error{job=~\"$job\",instance=~\"$node\"}) by (connector_name, source_id, error_msg)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{connector_name}}: {{error_msg}} ({{source_id}})","metric":"","query":"sum(cdc_source_error{job=~\"$job\",instance=~\"$node\"}) by (connector_name, source_id, error_msg)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CDC Source Errors","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming CDC","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":5},"height":null,"hideTimeOverride":false,"id":44,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"We first record the total blocking duration(ns) of output buffer of each actor. It shows how much time it takes an actor to process a message, i.e. a barrier, a watermark or rows of data, on average. Then we divide this duration by 1 second and show it as a percentage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":45,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, downstream_fragment_id) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}->{{downstream_fragment_id}}","metric":"","query":"avg(rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, downstream_fragment_id) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Output Blocking Time Ratio (Backpressure)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":46,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_actor_input_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, upstream_fragment_id) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}<-{{upstream_fragment_id}}","metric":"","query":"avg(rate(stream_actor_input_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, upstream_fragment_id) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Input Blocking Time Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":47,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_actor_in_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, upstream_fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}<-{{upstream_fragment_id}}","metric":"","query":"sum(rate(stream_actor_in_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, upstream_fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_in_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}}","metric":"","query":"rate(stream_actor_in_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Input Throughput (rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":48,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_actor_out_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_actor_out_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_out_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}}","metric":"","query":"rate(stream_actor_out_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Output Throughput (rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The operator-level memory usage statistics collected by each LRU cache","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":49,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (table_id, desc)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table {{table_id}} desc: {{desc}}","metric":"","query":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (table_id, desc)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_memory_usage{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table {{table_id}} actor {{actor_id}} desc: {{desc}}","metric":"","query":"stream_memory_usage{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Memory Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Memory usage aggregated by materialized views","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":50,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id) group_left(materialized_view_id) table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized view {{materialized_view_id}}","metric":"","query":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id) group_left(materialized_view_id) table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Memory Usage of Materialized Views","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":51,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"temporal join cache miss, table_id {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"temporal join cache miss, table_id {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Temporal Join Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":52,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_materialize_cache_hit_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache hit count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_materialize_cache_hit_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_materialize_cache_total_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total cached count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_materialize_cache_total_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_hit_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache hit count - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_materialize_cache_hit_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_total_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total cached count - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_materialize_cache_total_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialize Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":53,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache lookup count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_over_window_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_cache_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache miss count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_over_window_cache_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_over_window_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache lookup count - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_over_window_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_over_window_cache_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache miss count - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_over_window_cache_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_range_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"partition range cache lookup count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_over_window_range_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_range_cache_left_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"partition range cache left miss count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_over_window_range_cache_left_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_range_cache_right_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"partition range cache right miss count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_over_window_range_cache_right_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Over Window Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":54,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, fragment_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Join executor cache miss ratio - - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} fragment {{fragment_id}}","metric":"","query":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, fragment_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n appendonly cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream lookup cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream temporal join cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Materialize executor cache miss ratio - table {{table_id}} fragment {{fragment_id}}  {{instance}}","metric":"","query":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_over_window_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Over window cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_over_window_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_over_window_range_cache_left_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_range_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Over window partition range cache left miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_over_window_range_cache_left_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_range_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_over_window_range_cache_right_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_range_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Over window partition range cache right miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_over_window_range_cache_right_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_range_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Miss Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":55,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_barrier_align_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000) by (fragment_id,wait_side, executor)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}} {{wait_side}} {{executor}}","metric":"","query":"avg(rate(stream_barrier_align_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000) by (fragment_id,wait_side, executor)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_barrier_align_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}} fragment {{fragment_id}} {{wait_side}} {{executor}}","metric":"","query":"rate(stream_barrier_align_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Barrier Align Per Second","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":56,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - fragment {{fragment_id}} - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - fragment {{fragment_id}} - {{job}}","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p999 - fragment {{fragment_id}} - {{job}}","metric":"","query":"histogram_quantile(0.999, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - fragment {{fragment_id}} - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, fragment_id, job)(rate(stream_merge_barrier_align_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le,fragment_id,job) (rate(stream_merge_barrier_align_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - fragment {{fragment_id}} - {{job}}","metric":"","query":"sum by(le, fragment_id, job)(rate(stream_merge_barrier_align_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le,fragment_id,job) (rate(stream_merge_barrier_align_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Merger Barrier Align","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":57,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_join_actor_input_waiting_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000) by (fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}","metric":"","query":"avg(rate(stream_join_actor_input_waiting_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000) by (fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_actor_input_waiting_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}}","metric":"","query":"rate(stream_join_actor_input_waiting_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Actor Input Blocking Time Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":58,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_join_match_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000) by (fragment_id,side)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}} {{side}}","metric":"","query":"avg(rate(stream_join_match_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000) by (fragment_id,side)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_match_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}} {{side}}","metric":"","query":"rate(stream_join_match_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Actor Match Duration Per Second","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Multiple rows with distinct primary keys may have the same join key. This metric counts the number of join keys in the executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":59,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (fragment_id, side)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}} {{side}}","metric":"","query":"sum(stream_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (fragment_id, side)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}} {{side}}","metric":"","query":"stream_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of matched rows on the opposite side","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":60,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - fragment {{fragment_id}} table_id {{table_id}} - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - fragment {{fragment_id}} table_id {{table_id}} - {{job}}","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - fragment {{fragment_id}} table_id {{table_id}} - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, fragment_id, table_id) (rate(stream_join_matched_join_keys_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, fragment_id, table_id) (rate(stream_join_matched_join_keys_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - fragment {{fragment_id}} table_id {{table_id}} - {{job}}","metric":"","query":"sum by(le, job, fragment_id, table_id) (rate(stream_join_matched_join_keys_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, fragment_id, table_id) (rate(stream_join_matched_join_keys_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) >= 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Executor Matched Rows","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":61,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_agg_chunk_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level cache miss - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_agg_chunk_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_agg_chunk_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level total lookups - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_agg_chunk_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_chunk_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level cache miss  - table {{table_id}} actor {{actor_id}}}","metric":"","query":"rate(stream_agg_chunk_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_chunk_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level total lookups  - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_chunk_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Executor Cache Statistics For Each StreamChunk","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in each hash aggregation executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":62,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_agg_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg cached keys count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_agg_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_agg_distinct_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg distinct cached keys count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_agg_distinct_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg cached keys count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_agg_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_distinct_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg distinct cached keys count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_agg_distinct_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of dirty (unflushed) groups in each hash aggregation executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":63,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_agg_dirty_groups_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_agg_dirty_groups_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_dirty_groups_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_agg_dirty_groups_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Dirty Groups Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The total heap size of dirty (unflushed) groups in each hash aggregation executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":64,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_agg_dirty_groups_heap_size{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups heap size | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_agg_dirty_groups_heap_size{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_dirty_groups_heap_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups heap size | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_agg_dirty_groups_heap_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Dirty Groups Heap Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in each top_n executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":65,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_group_top_n_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_group_top_n_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_group_top_n_appendonly_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n appendonly cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_group_top_n_appendonly_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_group_top_n_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_group_top_n_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_group_top_n_appendonly_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n appendonly cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_group_top_n_appendonly_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"TopN Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in temporal join executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":66,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_temporal_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal Join cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_temporal_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_temporal_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal Join cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_temporal_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Temporal Join Cache Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in lookup executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":88},"height":null,"hideTimeOverride":false,"id":67,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_lookup_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lookup cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_lookup_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_lookup_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lookup cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_lookup_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lookup Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in over window executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":88},"height":null,"hideTimeOverride":false,"id":68,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_over_window_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"over window cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_over_window_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_over_window_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"over window cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_over_window_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_over_window_range_cache_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"over window partition range cache entry count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_over_window_range_cache_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Over Window Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"When enabled, this metric shows the input throughput of each executor.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":96},"height":null,"hideTimeOverride":false,"id":69,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_executor_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_identity, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_identity}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_executor_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_identity, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_executor_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_identity}} actor {{actor_id}}","metric":"","query":"rate(stream_executor_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The actor-level memory usage statistics reported by TaskLocalAlloc. (Disabled by default)","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":96},"height":null,"hideTimeOverride":false,"id":70,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(actor_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}","metric":"","query":"sum(actor_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"actor_memory_usage{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}}","metric":"","query":"actor_memory_usage{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Memory Usage (TaskLocalAlloc)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Actors","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":6},"height":null,"hideTimeOverride":false,"id":71,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":72,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_actor_execution_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_actor_execution_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Execution Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":73,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":74,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":8},"height":null,"hideTimeOverride":false,"id":75,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":8},"height":null,"hideTimeOverride":false,"id":76,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":77,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":16},"height":null,"hideTimeOverride":false,"id":78,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":16},"height":null,"hideTimeOverride":false,"id":79,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":80,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":24},"height":null,"hideTimeOverride":false,"id":81,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":24},"height":null,"hideTimeOverride":false,"id":82,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":83,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":32},"height":null,"hideTimeOverride":false,"id":84,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":32},"height":null,"hideTimeOverride":false,"id":85,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":86,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":40},"height":null,"hideTimeOverride":false,"id":87,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Avg Time","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Actors (Tokio)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":7},"height":null,"hideTimeOverride":false,"id":88,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":89,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{up_fragment_id}}->{{down_fragment_id}}","metric":"","query":"rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fragment-level Remote Exchange Send Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":90,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{up_fragment_id}}->{{down_fragment_id}}","metric":"","query":"rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fragment-level Remote Exchange Recv Throughput","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Exchange","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":91,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Errors that happened during computation. Check the logs for detailed error message.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":92,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_compute_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, executor_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{executor_name}} (fragment_id={{fragment_id}})","metric":"","query":"sum(user_compute_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, executor_name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compute Errors by Type","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Errors that happened during source data ingestion. Check the logs for detailed error message.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":93,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_source_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{source_name}} (source_id={{source_id}} fragment_id={{fragment_id}})","metric":"","query":"sum(user_source_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, source_id, source_name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Errors by Type","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Errors that happened during data sink out. Check the logs for detailed error message.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":94,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_sink_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, sink_id, sink_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{sink_name}} (sink_id={{sink_id}} fragment_id={{fragment_id}})","metric":"","query":"sum(user_sink_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, sink_id, sink_name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Errors by Type","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"User Streaming Errors","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":9},"height":null,"hideTimeOverride":false,"id":95,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"row"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":96,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{query_id}} : {{source_stage_id}}.{{source_task_id}} -> {{target_stage_id}}.{{target_task_id}}","metric":"","query":"batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Exchange Recv Row Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":97,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_task_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"batch_task_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Mpp Task Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"All memory usage of batch executors in bytes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":98,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"compute_batch_total_mem{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"compute_batch_total_mem{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"frontend_batch_total_mem{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"frontend_batch_total_mem{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Mem Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":99,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_heartbeat_worker_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"batch_heartbeat_worker_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Heartbeat Worker Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":100,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Row SeqScan Next Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Disk throughputs of spilling-out in the bacth query engine","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":101,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(batch_spill_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}} @ {{instance}}","metric":"","query":"sum(rate(batch_spill_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(batch_spill_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}} @ {{instance}}","metric":"","query":"sum(rate(batch_spill_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Spill Throughput","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Batch Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":10},"height":null,"hideTimeOverride":false,"id":102,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":103,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{table_id}} @ {{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_sst_store_block_request_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total_meta_miss_count - {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Hummock has three parts of memory usage: 1. Meta Cache 2. Block CacheThis metric shows the real memory usage of each of these three caches.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":104,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta cache - {{job}} @ {{instance}}","metric":"","query":"avg(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"data cache - {{job}} @ {{instance}}","metric":"","query":"avg(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(state_store_prefetch_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"prefetch cache - {{job}} @ {{instance}}","metric":"","query":"avg(state_store_prefetch_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":105,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='meta_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta cache miss ratio - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"(sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='meta_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_sst_store_block_request_counts{type='data_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='data_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache miss ratio - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"(sum(rate(state_store_sst_store_block_request_counts{type='data_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='data_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) >= 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Miss Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of the estimated hit ratio of a block while in the block cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":106,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_max(histogram_quantile(0.1, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache efficienfy - p10 - {{job}} @ {{instance}}","metric":"","query":"clamp_max(histogram_quantile(0.1, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_max(histogram_quantile(0.25, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache efficienfy - p25 - {{job}} @ {{instance}}","metric":"","query":"clamp_max(histogram_quantile(0.25, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_max(histogram_quantile(0.5, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache efficienfy - p50 - {{job}} @ {{instance}}","metric":"","query":"clamp_max(histogram_quantile(0.5, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_max(histogram_quantile(0.75, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache efficienfy - p75 - {{job}} @ {{instance}}","metric":"","query":"clamp_max(histogram_quantile(0.75, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_max(histogram_quantile(0.9, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache efficienfy - p90 - {{job}} @ {{instance}}","metric":"","query":"clamp_max(histogram_quantile(0.9, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_max(histogram_quantile(1.0, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache efficienfy - p100 - {{job}} @ {{instance}}","metric":"","query":"clamp_max(histogram_quantile(1.0, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Block Cache Efficiency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":107,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_scan_key_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type, table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter keys flow - {{table_id}} @ {{type}} @ {{instance}}","metric":"","query":"sum(rate(state_store_iter_scan_key_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type, table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iter keys flow","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":108,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts p50 - {{table_id}} @ {{job}} @ {{type}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts p99 - {{table_id}} @ {{job}} @ {{type}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts pmax - {{table_id}} @ {{job}} @ {{type}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts avg  - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Merged SSTs","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of the latency of Get operations that have been issued to the state store.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":109,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_get_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, table_id)(rate(state_store_get_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Duration - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of the time spent on iterator initialization.Histogram of the time spent on iterator scanning.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":110,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time p50 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time p99 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time pmax - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_iter_init_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, iter_type) (rate(state_store_iter_init_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time avg - {{iter_type}} {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(state_store_iter_init_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, iter_type) (rate(state_store_iter_init_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time p50 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time p99 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time pmax - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_iter_scan_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, iter_type) (rate(state_store_iter_scan_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time avg - {{iter_type}} {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(state_store_iter_scan_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, iter_type) (rate(state_store_iter_scan_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Duration - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":111,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter false positive count  - {{table_id}} - {{type}}","metric":"","query":"sum(irate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter positive count - {{table_id}} - {{type}}","metric":"","query":"sum(irate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter check count- {{table_id}} - {{type}}","metric":"","query":"sum(irate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom Filter Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Positive / Total","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":112,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter positive rate - {{table_id}} - {{type}}","metric":"","query":"(sum(rate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)) >= 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom Filter Positive Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"False-Positive / Total","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":113,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(((sum(rate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read req bloom filter false positive rate - {{table_id}} - {{type}}","metric":"","query":"(((sum(rate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))) >= 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom Filter False-Positive Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":114,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_iter_slow_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"state_store_iter_slow_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Slow Fetch Meta Unhits","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":115,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_shared_buffer_hit_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"shared_buffer hit - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_get_shared_buffer_hit_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id, iter_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{iter_type}} - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_iter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id, iter_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":116,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Size - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":117,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Size - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":118,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read p50 - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read p99 - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read pmax - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Read Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":119,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_iter_in_progress_counts{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Existing {{iter_type}} count @ {{table_id}}","metric":"","query":"state_store_iter_in_progress_counts{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_log_op_type_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, op_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter_log op count @ {{table_id}} {{op_type}}","metric":"","query":"sum(rate(state_store_iter_log_op_type_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, op_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Count - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size of a single key-value pair when reading by operation Get.Operation Get gets a single key-value pair with respect to a caller-specified key. If the key does not exist in the storage, the size of key is counted into this metric and the size of value is 0.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":120,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance) + sum(rate(state_store_get_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_get_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance) + sum(rate(state_store_get_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Throughput - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size of all the key-value paris when reading by operation Iter.Operation Iter scans a range of key-value pairs.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":121,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_iter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Throughput - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":122,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration avg - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fetch Meta Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":123,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_iter_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"state_store_iter_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fetch Meta Unhits","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock (Read)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":11},"height":null,"hideTimeOverride":false,"id":124,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric shows the real memory usage of uploader.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":125,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploading memory - {{job}} @ {{instance}}","metric":"","query":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_uploading_task_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploading task size - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_uploader_uploading_task_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_imm_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploader imm size - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_uploader_imm_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_imm_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance) - sum(state_store_uploader_uploading_task_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unflushed imm size - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_uploader_imm_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance) - sum(state_store_uploader_uploading_task_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance) - sum(state_store_uploader_imm_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"orphan imm size - {{job}} @ {{instance}}","metric":"","query":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance) - sum(state_store_uploader_imm_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_old_value_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"old value size - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_old_value_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader Memory Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of time spent on compacting shared buffer to remote storage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":126,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 Sync duration - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 Sync duration - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax Sync duration - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_sync_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg Sync duration - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance) (rate(state_store_sync_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_uploader_upload_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 upload task duration - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_uploader_upload_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_uploader_upload_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 upload task duration - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_uploader_upload_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_uploader_upload_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax upload task duration - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_uploader_upload_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Build and Sync Sstable Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":127,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.5, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write p50 - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(0.5, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.99, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write p99 - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(0.99, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write pmax - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Write Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":128,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_merge_imm_task_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"merge imm tasks - {{table_id}} @ {{instance}}","metric":"","query":"sum(irate(state_store_merge_imm_task_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_spill_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Uploader spill tasks - {{uploader_stage}} @ {{instance}}","metric":"","query":"sum(irate(state_store_spill_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_uploading_task_count{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploading task count - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_uploader_uploading_task_count{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_syncing_epoch_count{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"syncing epoch count - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_uploader_syncing_epoch_count{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader - Tasks Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":129,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_merge_imm_memory_sz{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Merging tasks memory size - {{table_id}} @ {{instance}}","metric":"","query":"sum(rate(state_store_merge_imm_memory_sz{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_spill_task_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Uploading tasks size - {{uploader_stage}} @ {{instance}}","metric":"","query":"sum(rate(state_store_spill_task_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader - Task Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":130,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write batch - {{table_id}} @ {{job}} @ {{instance}} ","metric":"","query":"sum(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"l0 - {{job}} @ {{instance}} ","metric":"","query":"sum(rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":131,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer avg - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":132,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_write_batch_tuple_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write_batch_kv_pair_count - {{table_id}} @ {{instance}}","metric":"","query":"sum(irate(state_store_write_batch_tuple_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Item Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":133,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_write_batch_size_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id) / sum(rate(state_store_write_batch_size_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"shared_buffer - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_write_batch_size_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id) / sum(rate(state_store_write_batch_size_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_shared_buffer_to_sstable_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance) / sum(rate(compactor_shared_buffer_to_sstable_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sync - {{job}} @ {{instance}}","metric":"","query":"sum(rate(compactor_shared_buffer_to_sstable_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance) / sum(rate(compactor_shared_buffer_to_sstable_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric shows the statistics of mem_table size on flush. By default only max (p100) is shown.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":134,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_write_batch_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, table_id, job, instance) (rate(state_store_write_batch_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance) (rate(state_store_write_batch_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, table_id, job, instance) (rate(state_store_write_batch_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Batch Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":135,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_mem_table_spill_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mem table spill table id - {{table_id}} @ {{instance}}","metric":"","query":"sum(irate(state_store_mem_table_spill_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Mem Table Spill Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":136,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Checkpoint Sync Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":137,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_event_handler_pending_event{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"sum(state_store_event_handler_pending_event{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Event handler pending event number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":138,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_event_handler_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, event_type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 {{event_type}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_event_handler_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, event_type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_event_handler_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, event_type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 {{event_type}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_event_handler_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, event_type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_event_handler_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, event_type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax {{event_type}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_event_handler_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, event_type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_uploader_wait_poll_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 finished_task_wait_poll {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_uploader_wait_poll_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_uploader_wait_poll_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 finished_task_wait_poll {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_uploader_wait_poll_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_uploader_wait_poll_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax finished_task_wait_poll {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_uploader_wait_poll_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Event handle latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock (Write)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":12},"height":null,"hideTimeOverride":false,"id":139,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of SSTables at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":140,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","query":"sum(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"SSTable Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size(KB) of SSTables at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":141,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","query":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"SSTable Size(KB)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The  of bytes that have been written by commit epoch per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":142,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_commit_write_throughput{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{table_id}}","metric":"","query":"sum(rate(storage_commit_write_throughput{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Commit Flush Bytes by Table","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have completed or failed","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":143,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_frequency{result!='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task_type}} - {{result}} - group-{{group}} @ {{compactor}}","metric":"","query":"sum(storage_level_compact_frequency{result!='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Failure Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have completed or failed","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":144,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_frequency{result='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task_type}} - {{result}} - group-{{group}} @ {{compactor}}","metric":"","query":"sum(storage_level_compact_frequency{result='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Success Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have been skipped.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":145,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_skip_compact_frequency{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (level, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{level}}-{{type}}","metric":"","query":"sum(rate(storage_skip_compact_frequency{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (level, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Skip Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Avg l0 select_level_count of the compact task, and categorize it according to different cg, levels and task types","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":146,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, group, type)(irate(storage_l0_compact_level_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, group, type)(irate(storage_l0_compact_level_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg cg{{group}}@{{type}}","metric":"","query":"sum by(le, group, type)(irate(storage_l0_compact_level_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, group, type)(irate(storage_l0_compact_level_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task L0 Select Level Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Avg file count of the compact task, and categorize it according to different cg, levels and task types","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":147,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, group, type)(irate(storage_compact_task_file_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, group, type)(irate(storage_compact_task_file_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg cg{{group}}@{{type}}","metric":"","query":"sum by(le, group, type)(irate(storage_compact_task_file_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, group, type)(irate(storage_compact_task_file_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task File Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The distribution of the compact task size triggered, including p90 and max. and categorize it according to different cg, levels and task types.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":148,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - cg{{group}}@{{type}}","metric":"","query":"histogram_quantile(0.9, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - cg{{group}}@{{type}}","metric":"","query":"histogram_quantile(1.0, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task Size Distribution","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that are running.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":149,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(storage_compact_task_pending_num{job=~\"$job\",instance=~\"$node\"}) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor_task_count - {{job}} @ {{instance}}","metric":"","query":"avg(storage_compact_task_pending_num{job=~\"$job\",instance=~\"$node\"}) by(job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(storage_compact_task_pending_parallelism{job=~\"$job\",instance=~\"$node\"}) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor_task_pending_parallelism - {{job}} @ {{instance}}","metric":"","query":"avg(storage_compact_task_pending_parallelism{job=~\"$job\",instance=~\"$node\"}) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compactor Running Task Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"compact-task: The total time have been spent on compaction.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":150,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task p50 - {{job}}","metric":"","query":"histogram_quantile(0.5, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task p90 - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task pmax - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range p90 - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range pmax - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get-table-id p90 - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get-table-id pmax - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io p90 - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io pmax - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(compute_refill_cache_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compute_apply_version_duration_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(compute_refill_cache_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le)(rate(compactor_compact_task_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(compactor_compact_task_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task avg","metric":"","query":"sum by(le)(rate(compactor_compact_task_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(compactor_compact_task_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le)(rate(state_store_compact_sst_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(state_store_compact_sst_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range avg","metric":"","query":"sum by(le)(rate(state_store_compact_sst_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(state_store_compact_sst_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"KBs read from next level during history compactions to next level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":151,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job) + sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}}","metric":"","query":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job) + sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}}","metric":"","query":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"flush - {{job}}","metric":"","query":"sum(rate(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_fast_compact_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fast compact - {{job}}","metric":"","query":"sum(rate(compactor_fast_compact_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of bytes that have been written by compaction.Flush refers to the process of compacting Memtables to SSTables at Level 0.Write refers to the process of compacting SSTables at one level to another level.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":152,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}}","metric":"","query":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"flush - {{job}}","metric":"","query":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Write Bytes(GiB)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Write amplification is the amount of bytes written to the remote storage by compaction for each one byte of flushed SSTable data. Write amplification is by definition higher than 1.0 because we write each piece of data to L0, and then write it again to an SSTable, and then compaction may read this piece of data and write it to a new SSTable, that's another write.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":153,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) / sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write amplification","metric":"","query":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) / sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Write Amplification","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of SSTables that is being compacted at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":154,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_level_compact_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","query":"storage_level_compact_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compacting SSTable Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"num of compact_task","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":155,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_level_compact_task_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task}}","metric":"","query":"storage_level_compact_task_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compacting Task Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":156,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from next level","metric":"","query":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from current level","metric":"","query":"sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} write to next level","metric":"","query":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"KBs Read/Write by Level","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":157,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_write_sstn{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} write to next level","metric":"","query":"sum(irate(storage_level_compact_write_sstn{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_read_sstn_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from next level","metric":"","query":"sum(irate(storage_level_compact_read_sstn_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_read_sstn_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from current level","metric":"","query":"sum(irate(storage_level_compact_read_sstn_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Count of SSTs Read/Write by level","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"For observing bloom_filter size, sstable file size, sstable block size etc.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":158,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.50, sum(rate(compactor_sstable_bloom_filter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom_filter_size_p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.50, sum(rate(compactor_sstable_bloom_filter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(rate(compactor_sstable_bloom_filter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom_filter_size_p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(rate(compactor_sstable_bloom_filter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Bloom Filter Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"For observing sstable file size","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":159,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.50, sum(rate(compactor_sstable_file_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sstable_file_size_p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.50, sum(rate(compactor_sstable_file_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(rate(compactor_sstable_file_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sstable_file_size_p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(rate(compactor_sstable_file_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable File Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"For observing sstable block size","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":160,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.50, sum(rate(compactor_sstable_block_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sstable_block_size_p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.50, sum(rate(compactor_sstable_block_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(rate(compactor_sstable_block_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sstable_block_size_p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(rate(compactor_sstable_block_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Block Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"For observing avg key and value count","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":161,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_key_size - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_value_size - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Avg Key And Value Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total time of operations which read from remote storage when enable prefetch","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":88},"height":null,"hideTimeOverride":false,"id":162,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io p90 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Remote Read Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":88},"height":null,"hideTimeOverride":false,"id":163,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_iter_scan_key_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter keys flow - {{type}} @ {{instance}} ","metric":"","query":"sum(rate(compactor_iter_scan_key_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compactor Iter keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"bytes of Lsm tree needed to reach balance","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":96},"height":null,"hideTimeOverride":false,"id":164,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_compact_pending_bytes{job=~\"$job\",instance=~\"$node\"}) by (instance, group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact pending bytes - {{group}} @ {{instance}} ","metric":"","query":"sum(storage_compact_pending_bytes{job=~\"$job\",instance=~\"$node\"}) by (instance, group)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lsm Compact Pending Bytes","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"compression ratio of each level of the lsm tree","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":96},"height":null,"hideTimeOverride":false,"id":165,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_compact_level_compression_ratio{job=~\"$job\",instance=~\"$node\"}) by (instance, group, level, algorithm)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lsm compression ratio - cg{{group}} @ L{{level}} - {{algorithm}} {{instance}}","metric":"","query":"sum(storage_compact_level_compression_ratio{job=~\"$job\",instance=~\"$node\"}) by (instance, group, level, algorithm)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lsm Level Compression Ratio","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Compaction","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":13},"height":null,"hideTimeOverride":false,"id":166,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":167,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":168,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, type, job, instance)(rate(object_store_operation_latency_sum{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(object_store_operation_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, type, job, instance)(rate(object_store_operation_latency_sum{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(object_store_operation_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":169,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type!~'streaming_upload_write_bytes|streaming_read_read_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_operation_latency_count{type!~'streaming_upload_write_bytes|streaming_read_read_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type=~'upload|delete',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{media_type}}-write - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_operation_latency_count{type=~'upload|delete',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type=~'read|readv|list|metadata',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{media_type}}-read - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_operation_latency_count{type=~'read|readv|list|metadata',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":170,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":171,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Failure Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":172,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_request_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_request_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Retry Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"There are two types of operations: 1. GET, SELECT, and DELETE, they cost 0.0004 USD per 1000 requests. 2. PUT, COPY, POST, LIST, they cost 0.005 USD per 1000 requests.Reading from S3 across different regions impose extra cost. This metric assumes 0.01 USD per 1GB data transfer. Please checkout AWS's pricing model for more accurate calculation.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"$"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":173,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}) * 0.01 / 1000 / 1000 / 1000","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"(Cross Region) Data Transfer Cost","metric":"","query":"sum(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}) * 0.01 / 1000 / 1000 / 1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_operation_latency_count{type=~'read|streaming_read_start|streaming_read_init',job=~\"$job\",instance=~\"$node\"}) * 0.0004 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GET, SELECT, and all other Requests Cost","metric":"","query":"sum(object_store_operation_latency_count{type=~'read|streaming_read_start|streaming_read_init',job=~\"$job\",instance=~\"$node\"}) * 0.0004 / 1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_operation_latency_count{type=~'upload|streaming_upload|streaming_upload_start|s3_upload_part|streaming_upload_finish|list',job=~\"$job\",instance=~\"$node\"}) * 0.005 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"PUT, COPY, POST, LIST Requests Cost","metric":"","query":"sum(object_store_operation_latency_count{type=~'upload|streaming_upload|streaming_upload_start|s3_upload_part|streaming_upload_finish|list',job=~\"$job\",instance=~\"$node\"}) * 0.005 / 1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Estimated S3 Cost (Realtime)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric uses the total size of data in S3 at this second to derive the cost of storing data for a whole month. The price is 0.023 USD per GB. Please checkout AWS's pricing model for more accurate calculation.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"$"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":174,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance) * 0.023 / 1000 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Monthly Storage Cost","metric":"","query":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance) * 0.023 / 1000 / 1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Estimated S3 Cost (Monthly)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Object Storage","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":14},"height":null,"hideTimeOverride":false,"id":175,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":176,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_hybrid_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - hybrid - {{op}} @ {{instance}}","metric":"","query":"sum(rate(foyer_hybrid_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hybrid Cache Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":177,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{name}} - hybrid - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{name}} - hybrid - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{name}} - hybrid - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{name}} - hybrid - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hybrid Cache Op Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":178,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_hybrid_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) / (sum(rate(foyer_hybrid_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) + sum(rate(foyer_hybrid_op_total{op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - hybrid - hit ratio @ {{instance}}","metric":"","query":"sum(rate(foyer_hybrid_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) / (sum(rate(foyer_hybrid_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) + sum(rate(foyer_hybrid_op_total{op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hybrid Cache Hit Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":179,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_memory_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - memory - {{op}} @ {{instance}}","metric":"","query":"sum(rate(foyer_memory_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Cache Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":180,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(foyer_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (name, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - memory - size @ {{instance}}","metric":"","query":"sum(foyer_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (name, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Cache Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":181,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_memory_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) / (sum(rate(foyer_memory_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) + sum(rate(foyer_memory_op_total{op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - memory - hit ratio @ {{instance}}","metric":"","query":"sum(rate(foyer_memory_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) / (sum(rate(foyer_memory_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) + sum(rate(foyer_memory_op_total{op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Cache Hit Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":182,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"sum(rate(foyer_storage_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":183,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_inner_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"sum(rate(foyer_storage_inner_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache Inner Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":184,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache Op Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":185,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache Inner Op Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":186,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) / (sum(rate(foyer_storage_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) + sum(rate(foyer_storage_op_total{op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - storage - hit ratio @ {{instance}}","metric":"","query":"sum(rate(foyer_storage_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) / (sum(rate(foyer_storage_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) + sum(rate(foyer_storage_op_total{op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache Hit Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":187,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(foyer_storage_region{job=~\"$job\",instance=~\"$node\"}) by (name, type, instance) * on(name, instance) group_left() avg(foyer_storage_region_size_bytes{job=~\"$job\",instance=~\"$node\"}) by (name, type, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - {{type}} region - size @ {{instance}}","metric":"","query":"sum(foyer_storage_region{job=~\"$job\",instance=~\"$node\"}) by (name, type, instance) * on(name, instance) group_left() avg(foyer_storage_region_size_bytes{job=~\"$job\",instance=~\"$node\"}) by (name, type, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Region Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":188,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_disk_io_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - disk - {{op}} @ {{instance}}","metric":"","query":"sum(rate(foyer_storage_disk_io_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Disk Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":189,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{name}} - disk - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{name}} - disk - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{name}} - disk - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{name}} - disk - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Disk Op Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":190,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_disk_io_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - disk - {{op}} @ {{instance}}","metric":"","query":"sum(rate(foyer_storage_disk_io_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Disk Op Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":191,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (type, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} file cache refill - {{op}} @ {{instance}}","metric":"","query":"sum(rate(refill_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (type, op, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=~\"meta|data\",op!~\"filtered|ignored\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (type, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} file cache refill - {{op}} @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=~\"meta|data\",op!~\"filtered|ignored\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (type, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":192,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} file cache - {{op}} @ {{instance}}","metric":"","query":"sum(rate(refill_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Data Refill Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":193,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{foyer}} cache refill - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{foyer}} cache refill - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{foyer}} cache refill - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{foyer}} cache refill - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":194,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(refill_queue_total) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"refill queue length @ {{instance}}","metric":"","query":"sum(refill_queue_total) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill Queue Length","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":195,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"parent_meta\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"parent meta lookup {{op}} @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=\"parent_meta\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inheritance - Parent Meta Lookup Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":196,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"parent_meta\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / (sum(rate(refill_total{type=\"parent_meta\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) + sum(rate(refill_total{type=\"parent_meta\",op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"parent meta lookup hit ratio @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=\"parent_meta\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / (sum(rate(refill_total{type=\"parent_meta\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) + sum(rate(refill_total{type=\"parent_meta\",op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)) >= 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inheritance - Parent Meta Lookup Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":197,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"unit_inheritance\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unit inheritance {{op}} @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=\"unit_inheritance\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inheritance - Unit inheritance Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":88},"height":null,"hideTimeOverride":false,"id":198,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"unit_inheritance\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / (sum(rate(refill_total{type=\"unit_inheritance\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) + sum(rate(refill_total{type=\"unit_inheritance\",op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unit inheritance ratio @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=\"unit_inheritance\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / (sum(rate(refill_total{type=\"unit_inheritance\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) + sum(rate(refill_total{type=\"unit_inheritance\",op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)) >= 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inheritance - Unit inheritance Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":88},"height":null,"hideTimeOverride":false,"id":199,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"block\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block refill {{op}} @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=\"block\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Block Refill Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":96},"height":null,"hideTimeOverride":false,"id":200,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"block\",op=\"success\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / sum(rate(refill_total{type=\"block\",op=\"unfiltered\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block refill ratio @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=\"block\",op=\"success\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / sum(rate(refill_total{type=\"block\",op=\"unfiltered\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) >= 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Block Refill Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Item numbers of the recent filter.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":96},"height":null,"hideTimeOverride":false,"id":201,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(recent_filter_items{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"items @ {{instance}}","metric":"","query":"sum(recent_filter_items{job=~\"$job\",instance=~\"$node\"}) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Recent Filter Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":104},"height":null,"hideTimeOverride":false,"id":202,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(recent_filter_ops{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recent filter {{op}} @ {{instance}}","metric":"","query":"sum(rate(recent_filter_ops{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Recent Filter Ops","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock Tiered Cache","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":15},"height":null,"hideTimeOverride":false,"id":203,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":204,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time p50 - {{method}} @ {{lock_type}} @ {{lock_name}}","metric":"","query":"histogram_quantile(0.5, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method, lock_name, lock_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time p99 - {{method}} @ {{lock_type}} @ {{lock_name}}","metric":"","query":"histogram_quantile(0.99, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method, lock_name, lock_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time pmax - {{method}} @ {{lock_type}} @ {{lock_name}}","metric":"","query":"histogram_quantile(1.0, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method, lock_name, lock_type))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lock Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":205,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time p50 - {{method}}","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time p99 - {{method}}","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time pmax - {{method}}","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Real Process Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":206,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version size","metric":"","query":"storage_version_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":207,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"current version id","metric":"","query":"storage_current_version_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"checkpoint version id","metric":"","query":"storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min pinned version id","metric":"","query":"storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_safepoint_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min safepoint version id","metric":"","query":"storage_min_safepoint_version_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Id","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":208,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"max committed epoch","metric":"","query":"storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_safe_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"safe epoch","metric":"","query":"storage_safe_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min pinned epoch","metric":"","query":"storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Epoch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":209,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_key_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","query":"storage_version_stats{metric='total_key_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_value_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","query":"storage_version_stats{metric='total_value_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Table Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":210,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_materialized_view_stats{metric='materialized_view_total_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{metric}}, mv id - {{table_id}} ","metric":"","query":"storage_materialized_view_stats{metric='materialized_view_total_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":211,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_key_count',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","query":"storage_version_stats{metric='total_key_count',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Table KV Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"\nObjects are classified into 3 groups:\n- not referenced by versions: these object are being deleted from object store.\n- referenced by non-current versions: these objects are stale (not in the latest version), but those old versions may still be in use (e.g. long-running pinning). Thus those objects cannot be deleted at the moment.\n- referenced by current version: these objects are in the latest version.\n\nAdditionally, a metric on all objects (including dangling ones) is updated with low-frequency. The metric is updated right before full GC. So subsequent full GC may reduce the actual value significantly, without updating the metric.\n                    ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":212,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","query":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","query":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","query":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_total_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all objects (including dangling ones)","metric":"","query":"storage_total_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Refer to `Object Total Number` panel for classification of objects.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":213,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","query":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","query":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","query":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_total_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all objects, including dangling ones","metric":"","query":"storage_total_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"total number of hummock version delta log","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":214,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_delta_log_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"delta log total number","metric":"","query":"storage_delta_log_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Delta Log Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"hummock version checkpoint latency","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":215,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(storage_version_checkpoint_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(storage_version_checkpoint_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_avg","metric":"","query":"rate(storage_version_checkpoint_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(storage_version_checkpoint_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Checkpoint Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"When certain per compaction group threshold is exceeded (e.g. number of level 0 sub-level in LSMtree), write op to that compaction group is stopped temporarily. Check log for detail reason of write stop.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":216,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_write_stop_compaction_groups{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compaction_group_{{compaction_group_id}}","metric":"","query":"storage_write_stop_compaction_groups{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Stop Compaction Groups","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"total number of attempts to trigger full GC","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":217,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_full_gc_trigger_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"full_gc_trigger_count","metric":"","query":"storage_full_gc_trigger_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Full GC Trigger Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"the object id watermark used in last full GC","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":218,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_full_gc_last_object_id_watermark{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"full_gc_last_object_id_watermark","metric":"","query":"storage_full_gc_last_object_id_watermark{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Full GC Last Watermark","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":219,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Event Loop Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The times of move_state_table occurs","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":220,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_move_state_table_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"move table cg{{group}}","metric":"","query":"sum(storage_move_state_table_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}) by (group)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Move State Table Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of state_tables in each CG","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":221,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_state_table_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"state table cg{{group}}","metric":"","query":"sum(irate(storage_state_table_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"State Table Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of branched_sst in each CG","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":222,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_branched_sst_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"branched sst cg{{group}}","metric":"","query":"sum(irate(storage_branched_sst_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Branched SST Count","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":223,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total backup job count since the Meta node starts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":224,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"backup_job_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"job count","metric":"","query":"backup_job_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Job Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Latency of backup jobs since the Meta node starts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":225,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time p50 - {{state}}","metric":"","query":"histogram_quantile(0.5, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time p99 - {{state}}","metric":"","query":"histogram_quantile(0.99, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time pmax - {{state}}","metric":"","query":"histogram_quantile(1.0, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Job Process Time","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Backup Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":17},"height":null,"hideTimeOverride":false,"id":226,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":227,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":228,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Drop latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":229,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"GetCatalog latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Catalog Service","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":18},"height":null,"hideTimeOverride":false,"id":230,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":231,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"AddWorkerNode latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":232,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"ListAllNodes latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Cluster Service","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":19},"height":null,"hideTimeOverride":false,"id":233,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":234,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CreateMaterializedView latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":235,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"DropMaterializedView latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":236,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Flush latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Stream Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":20},"height":null,"hideTimeOverride":false,"id":237,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":238,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UnpinVersionBefore latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":239,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UnpinSnapshotBefore latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":240,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"ReportCompactionTasks latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":241,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"GetNewSstIds latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Hummock Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":21},"height":null,"hideTimeOverride":false,"id":242,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":243,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_report_compaction_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_counts - {{instance}}","metric":"","query":"sum(irate(state_store_report_compaction_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"compaction_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":244,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_version_before_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_version_before_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_avg","metric":"","query":"sum(irate(state_store_unpin_version_before_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_version_before_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"version_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":245,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latencyp90 - {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_pin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_pin_snapshot_latency_count[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_avg","metric":"","query":"sum(irate(state_store_pin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_pin_snapshot_latency_count[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_snapshot_latency_count[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_avg","metric":"","query":"sum(irate(state_store_unpin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_snapshot_latency_count[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_unpin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(irate(state_store_unpin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"snapshot_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":246,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_pin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_counts - {{instance}}","metric":"","query":"sum(irate(state_store_pin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_counts - {{instance}}","metric":"","query":"sum(irate(state_store_unpin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"snapshot_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":247,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_get_new_sst_ids_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_get_new_sst_ids_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_avg","metric":"","query":"sum(irate(state_store_get_new_sst_ids_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_get_new_sst_ids_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"table_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":248,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_get_new_sst_ids_latency_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_counts - {{instance}}","metric":"","query":"sum(irate(state_store_get_new_sst_ids_latency_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"table_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":249,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_report_compaction_task_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_report_compaction_task_latency_count[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_avg","metric":"","query":"sum(irate(state_store_report_compaction_task_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_report_compaction_task_latency_count[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"compaction_latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC: Hummock Meta Client","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":22},"height":null,"hideTimeOverride":false,"id":250,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of active sessions","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":251,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Active Sessions","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":252,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Per Second (Local Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":253,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Per Second (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":254,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of running query in distributed execution mode","metric":"","query":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Running Queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":255,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of rejected query in distributed execution mode","metric":"","query":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Rejected queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":256,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of completed query in distributed execution mode","metric":"","query":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Completed Queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":257,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":258,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency (Local Query Mode)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Frontend","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":23},"height":null,"hideTimeOverride":false,"id":259,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":260,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(lru_runtime_loop_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"rate(lru_runtime_loop_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager loop count per sec","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":261,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_eviction_policy{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"lru_eviction_policy{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager eviction policy","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":262,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_latest_sequence{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"lru_latest_sequence{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_watermark_sequence{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"lru_watermark_sequence{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager sequence","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":263,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jemalloc_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The allocated memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":264,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_active_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jemalloc_active_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The active memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":265,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_resident_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jemalloc_resident_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The resident memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":266,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_metadata_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jemalloc_metadata_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The metadata memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":267,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jvm_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jvm_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The allocated memory of jvm","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":268,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jvm_active_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jvm_active_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The active memory of jvm","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":269,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_current_watermark_time_ms{job=~\"$job\",instance=~\"$node\"} - on() group_right() lru_evicted_watermark_time_ms{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table {{table_id}} actor {{actor_id}} desc: {{desc}}","metric":"","query":"lru_current_watermark_time_ms{job=~\"$job\",instance=~\"$node\"} - on() group_right() lru_evicted_watermark_time_ms{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager diff between current watermark and evicted watermark time (ms) for actors","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Memory manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":270,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":271,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(source_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_type}} @ {{source_id}}","metric":"","query":"rate(source_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Source Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":272,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(connector_sink_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector_type}} @ {{sink_id}}","metric":"","query":"rate(connector_sink_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Sink Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Connector Node","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":25},"height":null,"hideTimeOverride":false,"id":273,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":274,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id, sink_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 @ {{sink_id}} {{sink_name}} ({{connector}})","metric":"","query":"histogram_quantile(0.5, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id, sink_name))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id, sink_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 @ {{sink_id}} {{sink_name}} ({{connector}})","metric":"","query":"histogram_quantile(0.99, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id, sink_name))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id, sink_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax @ {{sink_id}} {{sink_name}} ({{connector}})","metric":"","query":"histogram_quantile(1.0, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id, sink_name))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, type, job, instance, sink_id, sink_name)(rate(sink_commit_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance, sink_id, sink_name) (rate(sink_commit_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg @ {{sink_id}} {{sink_name}} ({{connector}})","metric":"","query":"sum by(le, type, job, instance, sink_id, sink_name)(rate(sink_commit_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance, sink_id, sink_name) (rate(sink_commit_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Commit Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":275,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"log_store_latest_write_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"latest write epoch @ {{sink_id}} {{sink_name}} ({{connector}}) actor {{actor_id}}","metric":"","query":"log_store_latest_write_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"latest read epoch @ {{sink_id}} {{sink_name}} ({{connector}}) actor {{actor_id}}","metric":"","query":"log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"kv_log_store_buffer_unconsumed_min_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Kv log store uncomsuned min epoch @ {{sink_id}} {{sink_name}} ({{connector}}) actor {{actor_id}}","metric":"","query":"kv_log_store_buffer_unconsumed_min_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Read/Write Epoch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":276,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(max(log_store_latest_write_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, actor_id, sink_name)- max(log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, actor_id, sink_name)) / (2^16) / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Consume lag @ {{sink_id}} {{sink_name}} ({{connector}}) actor {{actor_id}}","metric":"","query":"(max(log_store_latest_write_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, actor_id, sink_name)- max(log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, actor_id, sink_name)) / (2^16) / 1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Lag","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":277,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(log_store_reader_wait_new_future_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id, actor_id, sink_name) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Backpressure @ {{sink_id}} {{sink_name}} ({{connector}}) actor {{actor_id}}","metric":"","query":"avg(rate(log_store_reader_wait_new_future_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id, actor_id, sink_name) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Backpressure Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":278,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_min((max(log_store_first_write_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, actor_id, sink_name)- max(log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, actor_id, sink_name)) / (2^16) / 1000, 0)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Consume persistent log lag @ {{sink_id}} {{sink_name}} ({{connector}}) actor {{actor_id}}","metric":"","query":"clamp_min((max(log_store_first_write_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, actor_id, sink_name)- max(log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, actor_id, sink_name)) / (2^16) / 1000, 0)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Consume Persistent Log Lag","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":279,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_read_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} ({{connector}})","metric":"","query":"sum(rate(log_store_read_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id, sink_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Consume Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":280,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_read_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, connector, sink_id, actor_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} ({{connector}}) actor {{actor_id}} @ {{instance}}","metric":"","query":"sum(rate(log_store_read_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, connector, sink_id, actor_id, sink_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Log Store Consume Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":281,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_write_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{sink_id}} {{sink_name}} ({{connector}})","metric":"","query":"sum(rate(log_store_write_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id, sink_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Write Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":282,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_write_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, connector, sink_id, actor_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} ({{connector}}) actor {{actor_id}} {{instance}}","metric":"","query":"sum(rate(log_store_write_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, connector, sink_id, actor_id, sink_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Log Store Write Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":283,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_read_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (actor_id, connector, sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} actor {{actor_id}} ({{connector}})","metric":"","query":"sum(rate(kv_log_store_storage_read_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (actor_id, connector, sink_id, sink_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Read Storage Row Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":284,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_read_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (actor_id, connector, sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} actor {{actor_id}} ({{connector}})","metric":"","query":"sum(rate(kv_log_store_storage_read_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (actor_id, connector, sink_id, sink_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Read Storage Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":285,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_write_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (actor_id, connector, sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} actor {{actor_id}} ({{connector}})","metric":"","query":"sum(rate(kv_log_store_storage_write_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (actor_id, connector, sink_id, sink_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Write Storage Row Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":286,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_write_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (actor_id, connector, sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} actor {{actor_id}} ({{connector}})","metric":"","query":"sum(rate(kv_log_store_storage_write_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (actor_id, connector, sink_id, sink_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Write Storage Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":287,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"kv_log_store_buffer_unconsumed_item_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Unconsumed item count @ {{sink_id}} {{sink_name}} ({{connector}}) actor {{actor_id}}","metric":"","query":"kv_log_store_buffer_unconsumed_item_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"kv_log_store_buffer_unconsumed_row_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Unconsumed row count @ {{sink_id}} {{sink_name}} ({{connector}}) actor {{actor_id}}","metric":"","query":"kv_log_store_buffer_unconsumed_row_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"kv_log_store_buffer_unconsumed_epoch_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Unconsumed epoch count @ {{sink_id}} {{sink_name}} ({{connector}}) actor {{actor_id}}","metric":"","query":"kv_log_store_buffer_unconsumed_epoch_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Buffer State","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":288,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_rewind_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (actor_id, connector, sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} actor {{actor_id}} ({{connector}})","metric":"","query":"sum(rate(kv_log_store_rewind_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (actor_id, connector, sink_id, sink_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Rewind Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":289,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(kv_log_store_rewind_delay_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, actor_id, connector, sink_id, sink_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} actor {{actor_id}} ({{connector}})","metric":"","query":"histogram_quantile(1.0, sum(rate(kv_log_store_rewind_delay_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, actor_id, connector, sink_id, sink_name))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Rewind delay (second)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total size of chunks buffered in a barrier","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":290,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_sink_chunk_buffer_size{job=~\"$job\",instance=~\"$node\"}) by (sink_id, actor_id, sink_name) * on(actor_id) group_left(sink_name) sink_info{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}} - actor {{actor_id}}","metric":"","query":"sum(stream_sink_chunk_buffer_size{job=~\"$job\",instance=~\"$node\"}) by (sink_id, actor_id, sink_name) * on(actor_id) group_left(sink_name) sink_info{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Chunk Buffer Size","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Sink Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":26},"height":null,"hideTimeOverride":false,"id":291,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Kafka high watermark by source and partition and source latest message by partition, source and actor","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":292,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_kafka_high_watermark{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"high watermark: source={{source_id}} partition={{partition}}","metric":"","query":"source_kafka_high_watermark{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_latest_message_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"latest msg: source={{source_id}} partition={{partition}} actor_id={{actor_id}}","metric":"","query":"source_latest_message_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kafka high watermark and source latest message","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Current number of messages in producer queues","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":293,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_msg_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Count in Producer Queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Current total size of messages in producer queues","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":294,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_msg_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_msg_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Size in Producer Queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of messages transmitted (produced) to Kafka brokers","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":295,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_tx_msgs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_tx_msgs{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Produced Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of messages consumed, not including ignored messages (due to offset, etc), from Kafka brokers.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":296,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_rx_msgs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_rx_msgs{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Received Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages awaiting transmission to broker","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":297,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_outbuf_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_outbuf_msg_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Count Pending to Transmit (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages in-flight to broker awaiting response","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":298,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_waitresp_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_waitresp_msg_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inflight Message Count (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of transmission errors","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":299,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_tx_errs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_tx_errs{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Error Count When Transmitting (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of receive errors","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":300,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rx_errs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_rx_errs{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Error Count When Receiving (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of requests timed out","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":301,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_req_timeouts{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_req_timeouts{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Timeout Request Count (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Broker latency / round-trip time in milli seconds","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":302,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_avg{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_avg{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p75{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_p75{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p90{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_p90{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_p99{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"RTT (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Broker throttling time in milliseconds","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":303,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_avg{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_avg{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p75{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_p75{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p90{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_p90{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_p99{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Throttle Time (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Age of metadata from broker for this topic (milliseconds)","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":304,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_metadata_age{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}","metric":"","query":"rdkafka_topic_metadata_age{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Topic Metadata_age Age","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Batch sizes in bytes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":305,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_avg{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_avg{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p75{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_p75{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p90{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_p90{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_p99{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p99_99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_p99_99{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_out_of_range{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_out_of_range{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Batch message counts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_avg{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_avg{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p75{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_p75{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p90{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_p90{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_p99{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p99_99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_p99_99{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_out_of_range{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_out_of_range{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Topic Batch Messages","transformations":[],"transparent":false,"type":"timeseries"}],"timeFrom":null,"timeShift":null,"title":"Topic Batch Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages ready to be produced in transmit queue","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":306,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_xmit_msgq_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","query":"rdkafka_topic_partition_xmit_msgq_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message to be Transmitted","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of pre-fetched messages in fetch queue","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":307,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_fetchq_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","query":"rdkafka_topic_partition_fetchq_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message in pre fetch queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Next offset to fetch","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":308,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_next_offset{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","query":"rdkafka_topic_partition_next_offset{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Next offset to fetch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Last committed offset","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":309,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_committed_offset{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","query":"rdkafka_topic_partition_committed_offset{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Committed Offset","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Kafka Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":27},"height":null,"hideTimeOverride":false,"id":310,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":311,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} read @ {{instance}}","metric":"","query":"sum(rate(connection_read_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} write @ {{instance}}","metric":"","query":"sum(rate(connection_write_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Network throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":312,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} read @ {{instance}}","metric":"","query":"sum(rate(connection_read_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} write @ {{instance}}","metric":"","query":"sum(rate(connection_write_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"S3 throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":313,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} read @ {{instance}}","metric":"","query":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} write @ {{instance}}","metric":"","query":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} total read @ {{instance}}","metric":"","query":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} total write @ {{instance}}","metric":"","query":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"gRPC throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":314,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_io_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 {{op_type}} err[{{error_kind}}] @ {{instance}}","metric":"","query":"sum(irate(connection_io_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_io_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} grpc {{op_type}} err[{{error_kind}}] @ {{instance}}","metric":"","query":"sum(rate(connection_io_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_io_err_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} total {{op_type}} err[{{error_kind}}] @ {{instance}}","metric":"","query":"sum(rate(connection_io_err_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"IO error rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":315,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(connection_count{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 @ {{instance}}","metric":"","query":"sum(connection_count{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(connection_count{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}) by (job, instance, connection_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} @ {{instance}}","metric":"","query":"sum(connection_count{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}) by (job, instance, connection_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Existing connection count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":316,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_create_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 @ {{instance}}","metric":"","query":"sum(irate(connection_create_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_create_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} @ {{instance}}","metric":"","query":"sum(irate(connection_create_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create new connection rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":317,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 @ {{instance}}","metric":"","query":"sum(irate(connection_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} @ {{instance}}","metric":"","query":"sum(irate(connection_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create new connection err rate","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Network connection","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":28},"height":null,"hideTimeOverride":false,"id":318,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"iceberg write qps","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":319,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"iceberg_write_qps{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} actor {{actor_id}}","metric":"","query":"iceberg_write_qps{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Qps Of Iceberg Writer","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":320,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id, sink_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 @ {{sink_id}} {{sink_name}}","metric":"","query":"histogram_quantile(0.5, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id, sink_name))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id, sink_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 @ {{sink_id}} {{sink_name}}","metric":"","query":"histogram_quantile(0.99, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id, sink_name))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id, sink_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax @ {{sink_id}} {{sink_name}}","metric":"","query":"histogram_quantile(1.0, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id, sink_name))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, type, job, instance, sink_id, sink_name)(rate(iceberg_write_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance, sink_id, sink_name) (rate(iceberg_write_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg @ {{sink_id}} {{sink_name}}","metric":"","query":"sum by(le, type, job, instance, sink_id, sink_name)(rate(iceberg_write_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance, sink_id, sink_name) (rate(iceberg_write_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Latency Of Iceberg Writer","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":321,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"iceberg_rolling_unflushed_data_file{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} actor {{actor_id}}","metric":"","query":"iceberg_rolling_unflushed_data_file{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iceberg rolling unfushed data file","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":322,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"iceberg_position_delete_cache_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} actor {{actor_id}}","metric":"","query":"iceberg_position_delete_cache_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iceberg position delete cache num","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":323,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"iceberg_partition_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} actor {{actor_id}}","metric":"","query":"iceberg_partition_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iceberg partition num","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Iceberg Sink Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":29},"height":null,"hideTimeOverride":false,"id":324,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":325,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_success_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_success_count - {{instance}}","metric":"","query":"sum(rate(udf_success_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_failure_count - {{instance}}","metric":"","query":"sum(rate(udf_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_retry_count - {{instance}}","metric":"","query":"sum(rate(udf_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_success_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_success_count - {{link}} {{name}} {{fragment_id}}","metric":"","query":"sum(rate(udf_success_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_failure_count - {{link}} {{name}} {{fragment_id}}","metric":"","query":"sum(rate(udf_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_retry_count - {{instance}}","metric":"","query":"sum(rate(udf_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Calls Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":326,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(udf_input_chunk_rows_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / sum(irate(udf_input_chunk_rows_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_input_chunk_rows_avg - {{link}} {{name}} {{fragment_id}}","metric":"","query":"sum(irate(udf_input_chunk_rows_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / sum(irate(udf_input_chunk_rows_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Input Chunk Rows","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":327,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.50, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.50, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(udf_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / sum(irate(udf_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_avg - {{instance}}","metric":"","query":"sum(irate(udf_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / sum(irate(udf_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, link, name, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_p99_by_name - {{link}} {{name}} {{fragment_id}}","metric":"","query":"histogram_quantile(0.99, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, link, name, fragment_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(udf_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / sum(irate(udf_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_avg_by_name - {{link}} {{name}} {{fragment_id}}","metric":"","query":"sum(irate(udf_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / sum(irate(udf_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":328,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_input_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_throughput_rows - {{instance}}","metric":"","query":"sum(rate(udf_input_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_input_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_throughput_rows - {{link}} {{name}} {{fragment_id}}","metric":"","query":"sum(rate(udf_input_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Throughput (rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":329,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_throughput_bytes - {{instance}}","metric":"","query":"sum(rate(udf_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_throughput_bytes - {{link}} {{name}} {{fragment_id}}","metric":"","query":"sum(rate(udf_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / (1024*1024)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Throughput (bytes)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Currently only embedded JS UDF supports this. Others will always show 0.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":330,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(udf_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_memory_usage - {{instance}}","metric":"","query":"sum(udf_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(udf_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_memory_usage - {{name}} {{fragment_id}}","metric":"","query":"sum(udf_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Memory Usage (bytes)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"User Defined Function","transformations":[],"transparent":false,"type":"row"}],"refresh":"","rows":[],"schemaVersion":12,"sharedCrosshair":true,"style":"dark","tags":["risingwave"],"templating":{"list":[{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, instance)","description":"Reporting instance of the metric","hide":0,"includeAll":true,"label":"Node","multi":true,"name":"node","options":[],"query":{"query":"label_values(process_cpu_seconds_total, instance)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, job)","description":"Reporting job of the metric","hide":0,"includeAll":true,"label":"Job","multi":true,"name":"job","options":[],"query":{"query":"label_values(process_cpu_seconds_total, job)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(table_info, table_id)","description":"Reporting table id of the metric","hide":0,"includeAll":true,"label":"Table","multi":true,"name":"table","options":[],"query":{"query":"label_values(table_info, table_id)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"}]},"time":{"from":"now-30m","to":"now"},"timepicker":{"hidden":false,"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone":"browser","title":"risingwave_dev_dashboard","uid":"Ecy3uV1nz","version":0}
diff --git a/grafana/risingwave-user-dashboard.json b/grafana/risingwave-user-dashboard.json
index 037679704e38f..0d0619aad29ec 100644
--- a/grafana/risingwave-user-dashboard.json
+++ b/grafana/risingwave-user-dashboard.json
@@ -1 +1 @@
-{"__inputs":[],"annotations":{"list":[]},"description":"RisingWave Dashboard","editable":true,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"panels":[{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":1,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Information about actors","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":2,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true,"sortBy":[]},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"group(actor_info{job=~\"$job\",instance=~\"$node\"}) by (actor_id, fragment_id, compute_node)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"group(actor_info{job=~\"$job\",instance=~\"$node\"}) by (actor_id, fragment_id, compute_node)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true},"indexByName":{"actor_id":0,"compute_node":2,"fragment_id":1}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Information about state tables. Column `materialized_view_id` is the id of the materialized view that this state table belongs to.","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":3,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true,"sortBy":[]},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name, table_type, materialized_view_id, fragment_id, compaction_group_id)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name, table_type, materialized_view_id, fragment_id, compaction_group_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"State Table Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true},"indexByName":{"compaction_group_id":5,"fragment_id":4,"materialized_view_id":3,"table_id":0,"table_name":1,"table_type":2}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Actor count per compute node","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":4,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true,"sortBy":[]},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"count(actor_info{job=~\"$job\",instance=~\"$node\"}) by (compute_node)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"count(actor_info{job=~\"$job\",instance=~\"$node\"}) by (compute_node)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Count (Group By Compute Node)","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true},"indexByName":{"compaction_group_id":5,"fragment_id":4,"materialized_view_id":3,"table_id":0,"table_name":1,"table_type":2}}}],"transparent":false,"type":"table"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Actor/Table Id Info","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":false,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":1},"height":null,"hideTimeOverride":false,"id":5,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Overview","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":2},"height":null,"hideTimeOverride":false,"id":6,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","query":"sum(rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of bytes read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":2},"height":null,"hideTimeOverride":false,"id":7,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum by (source_id, source_name, fragment_id)(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","query":"(sum by (source_id, source_name, fragment_id)(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of rows streamed into each sink per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":10},"height":null,"hideTimeOverride":false,"id":8,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id) * on(sink_id) group_left(sink_name) group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}}","metric":"","query":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id) * on(sink_id) group_left(sink_name) group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (sink_id, sink_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized view per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":10},"height":null,"hideTimeOverride":false,"id":9,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id) * on(table_id) group_left(table_name) group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mview {{table_id}} {{table_name}}","metric":"","query":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id) * on(table_id) group_left(table_name) group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The time that the data between two consecutive barriers gets fully processed, i.e. the computation results are made durable into materialized views or sink to external systems. This metric shows to users the freshness of materialized views.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":18},"height":null,"hideTimeOverride":false,"id":10,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_avg","metric":"","query":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Alerts in the system group by type:\n            - Too Many Barriers: there are too many uncommitted barriers generated. This means the streaming graph is stuck or under heavy load. Check 'Barrier Latency' panel.\n            - Recovery Triggered: cluster recovery is triggered. Check 'Errors by Type' / 'Node Count' panels.\n            - Lagging Version: the checkpointed or pinned version id is lagging behind the current version id. Check 'Hummock Manager' section in dev dashboard.\n            - Lagging Epoch: the pinned or safe epoch is lagging behind the current max committed epoch. Check 'Hummock Manager' section in dev dashboard.\n            - Lagging Compaction: there are too many files in L0. This can be caused by compactor failure or lag of compactor resource. Check 'Compaction' section in dev dashboard.\n            - Lagging Vacuum: there are too many stale files waiting to be cleaned. This can be caused by compactor failure or lag of compactor resource. Check 'Compaction' section in dev dashboard.\n            - Abnormal Meta Cache Memory: the meta cache memory usage is too large, exceeding the expected 10 percent.\n            - Abnormal Block Cache Memory: the block cache memory usage is too large, exceeding the expected 10 percent.\n            - Abnormal Uploading Memory Usage: uploading memory is more than 70 percent of the expected, and is about to spill.\n            - Write Stall: Compaction cannot keep up. Stall foreground write.\n            ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":18},"height":null,"hideTimeOverride":false,"id":11,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"} >= bool 200","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Too Many Barriers","metric":"","query":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"} >= bool 200","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > bool 0 + sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) > bool 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Recovery Triggered","metric":"","query":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > bool 0 + sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) > bool 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"((storage_current_version_id{job=~\"$job\",instance=~\"$node\"} - storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}) >= bool 100) + ((storage_current_version_id{job=~\"$job\",instance=~\"$node\"} - storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}) >= bool 100)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Version","metric":"","query":"((storage_current_version_id{job=~\"$job\",instance=~\"$node\"} - storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}) >= bool 100) + ((storage_current_version_id{job=~\"$job\",instance=~\"$node\"} - storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}) >= bool 100)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"((storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"} - storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}) >= bool 6553600000 unless + storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"} == 0)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Epoch","metric":"","query":"((storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"} - storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}) >= bool 6553600000 unless + storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"} == 0)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(label_replace(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}, 'L0', 'L0', 'level_index', '.*_L0') unless storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (L0) >= bool 200","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Compaction","metric":"","query":"sum(label_replace(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}, 'L0', 'L0', 'level_index', '.*_L0') unless storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (L0) >= bool 200","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"} >= bool 200","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Vacuum","metric":"","query":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"} >= bool 200","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_meta_cache_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 1.1","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Abnormal Meta Cache Memory","metric":"","query":"state_store_meta_cache_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 1.1","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_block_cache_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 1.1","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Abnormal Block Cache Memory","metric":"","query":"state_store_block_cache_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 1.1","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_uploading_memory_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 0.7","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Abnormal Uploading Memory Usage","metric":"","query":"state_store_uploading_memory_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 0.7","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_write_stop_compaction_groups{job=~\"$job\",instance=~\"$node\"} > bool 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Write Stall","metric":"","query":"storage_write_stop_compaction_groups{job=~\"$job\",instance=~\"$node\"} > bool 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Alerts","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Errors in the system group by type","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":26},"height":null,"hideTimeOverride":false,"id":12,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_compute_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, executor_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{executor_name}} (fragment_id={{fragment_id}})","metric":"","query":"sum(user_compute_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, executor_name, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_source_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{source_name}} (source_id={{source_id}} fragment_id={{fragment_id}})","metric":"","query":"sum(user_source_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, source_id, source_name, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_sink_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, sink_id, sink_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{sink_name}} (sink_id={{sink_id}} fragment_id={{fragment_id}})","metric":"","query":"sum(user_sink_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, sink_id, sink_name, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_status_is_up{job=~\"$job\",instance=~\"$node\"} == 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source error: source_id={{source_id}}, source_name={{source_name}} @ {{instance}}","metric":"","query":"source_status_is_up{job=~\"$job\",instance=~\"$node\"} == 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote storage error {{type}}: {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Errors","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":26},"height":null,"hideTimeOverride":false,"id":13,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Local mode","metric":"","query":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distributed mode","metric":"","query":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Query QPS","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of each type of RisingWave components alive.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":34},"height":null,"hideTimeOverride":false,"id":14,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_type}}","metric":"","query":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of active sessions in frontend nodes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":34},"height":null,"hideTimeOverride":false,"id":15,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Active Sessions","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":42},"height":null,"hideTimeOverride":false,"id":16,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The CPU usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":17,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of CPU cores per RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":18,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU Core Number","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"CPU","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":43},"height":null,"hideTimeOverride":false,"id":19,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The memory usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":20,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Memory","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":21,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (instance) + sum(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (instance) + sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage @ {{instance}}","metric":"","query":"sum(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (instance) + sum(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (instance) + sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Usage (Total)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":22,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(actor_memory_usage[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"streaming actor - {{actor_id}}","metric":"","query":"rate(actor_memory_usage[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage meta cache - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage block cache - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage write buffer - {{job}} @ {{instance}}","metric":"","query":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id) group_left(materialized_view_id) table_info) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized_view {{materialized_view_id}}","metric":"","query":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id) group_left(materialized_view_id) table_info) by (materialized_view_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Usage (Detailed)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Executor cache statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":23,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Join - cache miss - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Join - total lookups - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n appendonly - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n appendonly - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lookup executor - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lookup executor - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal join - cache miss - table_id {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal join - total lookups - table_id {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Materialize - cache hit count - table {{table_id}} - actor {{actor_id}}  {{instance}}","metric":"","query":"rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Materialize - total cache count - table {{table_id}} - actor {{actor_id}}  {{instance}}","metric":"","query":"rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":24,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id)) >=0 ","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"join executor cache miss ratio - - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","query":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id)) >=0 ","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0 ","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0 ","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n appendonly cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream lookup cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream temporal join cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialize executor cache miss ratio - table {{table_id}} - actor {{actor_id}}  {{instance}}","metric":"","query":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Miss Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Storage cache statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":25,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"memory cache - {{table_id}} @ {{type}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_sst_store_block_request_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total_meta_miss_count - {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Storage bloom filter statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":26,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_read_req_check_bloom_filter_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter total - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_read_req_check_bloom_filter_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_read_req_positive_but_non_exist_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter false positive  - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_read_req_positive_but_non_exist_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Bloom Filer","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Storage file cache statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":27,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(file_cache_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"file cache {{op}} @ {{instance}}","metric":"","query":"sum(rate(file_cache_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(file_cache_miss{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"file cache miss @ {{instance}}","metric":"","query":"sum(rate(file_cache_miss{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage File Cache","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Memory","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":44},"height":null,"hideTimeOverride":false,"id":28,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Send/Recv throughput per node for streaming exchange","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":29,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Send @ {{instance}}","metric":"","query":"sum(rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Recv @ {{instance}}","metric":"","query":"sum(rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Streming Remote Exchange (Bytes/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The remote storage read/write throughput per node","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":30,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{instance}}","metric":"","query":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{instance}}","metric":"","query":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Remote I/O (Bytes/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"row"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":31,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{query_id}} : {{source_stage_id}}.{{source_task_id}} -> {{target_stage_id}}.{{target_task_id}}","metric":"","query":"batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Exchange Recv (Rows/s)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Network","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":45},"height":null,"hideTimeOverride":false,"id":32,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"\n                    Objects are classified into 3 groups:\n                    - not referenced by versions: these object are being deleted from object store.\n                    - referenced by non-current versions: these objects are stale (not in the latest version), but those old versions may still be in use (e.g. long-running pinning). Thus those objects cannot be deleted at the moment.\n                    - referenced by current version: these objects are in the latest version.\n                    ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":33,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","query":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","query":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","query":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The storage size of each materialized view","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":34,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_materialized_view_stats{metric='materialized_view_total_size',job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{metric}}, mv id - {{table_id}} ","metric":"","query":"storage_materialized_view_stats{metric='materialized_view_total_size',job=~\"$job\",instance=~\"$node\"}/1024","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"\n                    Objects are classified into 3 groups:\n                    - not referenced by versions: these object are being deleted from object store.\n                    - referenced by non-current versions: these objects are stale (not in the latest version), but those old versions may still be in use (e.g. long-running pinning). Thus those objects cannot be deleted at the moment.\n                    - referenced by current version: these objects are in the latest version.\n                    ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":35,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","query":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","query":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","query":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of bytes that have been written by compaction.Flush refers to the process of compacting Memtables to SSTables at Level 0.Compaction refers to the process of compacting SSTables at one level to another level.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":36,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Compaction - {{job}}","metric":"","query":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush - {{job}}","metric":"","query":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Bytes","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The remote storage read/write throughput","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":37,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}}","metric":"","query":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}}","metric":"","query":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Remote I/O (Bytes/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Size statistics for checkpoint","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":38,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{job}}","metric":"","query":"sum by(le, job) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Checkpoint Size","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Storage","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":46},"height":null,"hideTimeOverride":false,"id":39,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":40,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_name}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of bytes read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":41,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum by (source_id)(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_id}}","metric":"","query":"(sum by (source_id)(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":42,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_source_backfill_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","query":"sum(rate(stream_source_backfill_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Backfill Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized executor actor per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":43,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_executor_row_count{executor_identity=~\".*MaterializeExecutor.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) * on(actor_id) group_left(materialized_view_id, table_name) (group(table_info{table_type=~\"MATERIALIZED_VIEW\",job=~\"$job\",instance=~\"$node\"}) by (actor_id, materialized_view_id, table_name))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized view {{table_name}} table_id {{materialized_view_id}}","metric":"","query":"sum(rate(stream_executor_row_count{executor_identity=~\".*MaterializeExecutor.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) * on(actor_id) group_left(materialized_view_id, table_name) (group(table_info{table_type=~\"MATERIALIZED_VIEW\",job=~\"$job\",instance=~\"$node\"}) by (actor_id, materialized_view_id, table_name))) by (materialized_view_id, table_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been read from the backfill operator used by MV on MV","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":44,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_snapshot_read_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Read Snapshot - table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_backfill_snapshot_read_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_upstream_output_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Upstream - table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_backfill_upstream_output_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"We first record the total blocking duration(ns) of output buffer of each actor. It shows how much time it takes an actor to process a message, i.e. a barrier, a watermark or rows of data, on average. Then we divide this duration by 1 second and show it as a percentage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":45,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, downstream_fragment_id) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}->{{downstream_fragment_id}}","metric":"","query":"avg(rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, downstream_fragment_id) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Output Blocking Time Ratio (Backpressure)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":47},"height":null,"hideTimeOverride":false,"id":46,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":47,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of running query in distributed execution mode","metric":"","query":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Running query in distributed execution mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":48,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of rejected query in distributed execution mode","metric":"","query":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Rejected query in distributed execution mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":49,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of completed query in distributed execution mode","metric":"","query":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Completed query in distributed execution mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":50,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency in Distributed Execution Mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":51,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency in Local Execution Mode","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Batch","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":52,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":53,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(connector_source_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_type}} @ {{source_id}}","metric":"","query":"rate(connector_source_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Source Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":54,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(connector_sink_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector_type}} @ {{sink_id}}","metric":"","query":"rate(connector_sink_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Sink Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Connector Node","transformations":[],"transparent":false,"type":"row"}],"refresh":"","rows":[],"schemaVersion":12,"sharedCrosshair":true,"style":"dark","tags":["risingwave"],"templating":{"list":[{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, instance)","description":"Reporting instance of the metric","hide":0,"includeAll":true,"label":"Node","multi":true,"name":"node","options":[],"query":{"query":"label_values(process_cpu_seconds_total, instance)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, job)","description":"Reporting job of the metric","hide":0,"includeAll":true,"label":"Job","multi":true,"name":"job","options":[],"query":{"query":"label_values(process_cpu_seconds_total, job)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"}]},"time":{"from":"now-30m","to":"now"},"timepicker":{"hidden":false,"nowDelay":null,"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone":"browser","title":"risingwave_dashboard","uid":"Fcy3uV1nz","version":0}
+{"__inputs":[],"annotations":{"list":[]},"description":"RisingWave Dashboard","editable":true,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"panels":[{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":1,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Information about actors","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]}},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":2,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true},"repeat":null,"repeatDirection":null,"span":6,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"group(actor_info{job=~\"$job\",instance=~\"$node\"}) by (actor_id, fragment_id, compute_node)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"group(actor_info{job=~\"$job\",instance=~\"$node\"}) by (actor_id, fragment_id, compute_node)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true},"indexByName":{"actor_id":0,"compute_node":2,"fragment_id":1}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Information about state tables. Column `materialized_view_id` is the id of the materialized view that this state table belongs to.","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]}},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":3,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true},"repeat":null,"repeatDirection":null,"span":6,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name, table_type, materialized_view_id, fragment_id, compaction_group_id)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name, table_type, materialized_view_id, fragment_id, compaction_group_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"State Table Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true},"indexByName":{"compaction_group_id":5,"fragment_id":4,"materialized_view_id":3,"table_id":0,"table_name":1,"table_type":2}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Actor count per compute node","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]}},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":4,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true},"repeat":null,"repeatDirection":null,"span":6,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"count(actor_info{job=~\"$job\",instance=~\"$node\"}) by (compute_node)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"count(actor_info{job=~\"$job\",instance=~\"$node\"}) by (compute_node)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Count (Group By Compute Node)","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true},"indexByName":{"compaction_group_id":5,"fragment_id":4,"materialized_view_id":3,"table_id":0,"table_name":1,"table_type":2}}}],"transparent":false,"type":"table"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Actor/Table Id Info","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":false,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":1},"height":null,"hideTimeOverride":false,"id":5,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Overview","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":2},"height":null,"hideTimeOverride":false,"id":6,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","query":"sum(rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of bytes read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":2},"height":null,"hideTimeOverride":false,"id":7,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum by (source_id, source_name, fragment_id)(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","query":"(sum by (source_id, source_name, fragment_id)(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of rows streamed into each sink per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":10},"height":null,"hideTimeOverride":false,"id":8,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id) * on(sink_id) group_left(sink_name) group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}}","metric":"","query":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id) * on(sink_id) group_left(sink_name) group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (sink_id, sink_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized view per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":10},"height":null,"hideTimeOverride":false,"id":9,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id) * on(table_id) group_left(table_name) group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mview {{table_id}} {{table_name}}","metric":"","query":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id) * on(table_id) group_left(table_name) group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The time that the data between two consecutive barriers gets fully processed, i.e. the computation results are made durable into materialized views or sink to external systems. This metric shows to users the freshness of materialized views.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":18},"height":null,"hideTimeOverride":false,"id":10,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_avg","metric":"","query":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Alerts in the system group by type:\n            - Too Many Barriers: there are too many uncommitted barriers generated. This means the streaming graph is stuck or under heavy load. Check 'Barrier Latency' panel.\n            - Recovery Triggered: cluster recovery is triggered. Check 'Errors by Type' / 'Node Count' panels.\n            - Lagging Version: the checkpointed or pinned version id is lagging behind the current version id. Check 'Hummock Manager' section in dev dashboard.\n            - Lagging Epoch: the pinned or safe epoch is lagging behind the current max committed epoch. Check 'Hummock Manager' section in dev dashboard.\n            - Lagging Compaction: there are too many files in L0. This can be caused by compactor failure or lag of compactor resource. Check 'Compaction' section in dev dashboard.\n            - Lagging Vacuum: there are too many stale files waiting to be cleaned. This can be caused by compactor failure or lag of compactor resource. Check 'Compaction' section in dev dashboard.\n            - Abnormal Meta Cache Memory: the meta cache memory usage is too large, exceeding the expected 10 percent.\n            - Abnormal Block Cache Memory: the block cache memory usage is too large, exceeding the expected 10 percent.\n            - Abnormal Uploading Memory Usage: uploading memory is more than 70 percent of the expected, and is about to spill.\n            - Write Stall: Compaction cannot keep up. Stall foreground write.\n            ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":18},"height":null,"hideTimeOverride":false,"id":11,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"} >= bool 200","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Too Many Barriers","metric":"","query":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"} >= bool 200","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > bool 0 + sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) > bool 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Recovery Triggered","metric":"","query":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > bool 0 + sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) > bool 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"((storage_current_version_id{job=~\"$job\",instance=~\"$node\"} - storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}) >= bool 100) + ((storage_current_version_id{job=~\"$job\",instance=~\"$node\"} - storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}) >= bool 100)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Version","metric":"","query":"((storage_current_version_id{job=~\"$job\",instance=~\"$node\"} - storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}) >= bool 100) + ((storage_current_version_id{job=~\"$job\",instance=~\"$node\"} - storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}) >= bool 100)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"((storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"} - storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}) >= bool 6553600000 unless + storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"} == 0)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Epoch","metric":"","query":"((storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"} - storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}) >= bool 6553600000 unless + storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"} == 0)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(label_replace(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}, 'L0', 'L0', 'level_index', '.*_L0') unless storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (L0) >= bool 200","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Compaction","metric":"","query":"sum(label_replace(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}, 'L0', 'L0', 'level_index', '.*_L0') unless storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (L0) >= bool 200","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"} >= bool 200","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Vacuum","metric":"","query":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"} >= bool 200","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_meta_cache_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 1.1","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Abnormal Meta Cache Memory","metric":"","query":"state_store_meta_cache_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 1.1","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_block_cache_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 1.1","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Abnormal Block Cache Memory","metric":"","query":"state_store_block_cache_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 1.1","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_uploading_memory_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 0.7","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Abnormal Uploading Memory Usage","metric":"","query":"state_store_uploading_memory_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 0.7","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_write_stop_compaction_groups{job=~\"$job\",instance=~\"$node\"} > bool 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Write Stall","metric":"","query":"storage_write_stop_compaction_groups{job=~\"$job\",instance=~\"$node\"} > bool 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Alerts","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Errors in the system group by type","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":26},"height":null,"hideTimeOverride":false,"id":12,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_compute_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, executor_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{executor_name}} (fragment_id={{fragment_id}})","metric":"","query":"sum(user_compute_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, executor_name, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_source_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{source_name}} (source_id={{source_id}} fragment_id={{fragment_id}})","metric":"","query":"sum(user_source_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, source_id, source_name, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_sink_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, sink_id, sink_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{sink_name}} (sink_id={{sink_id}} fragment_id={{fragment_id}})","metric":"","query":"sum(user_sink_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, sink_id, sink_name, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_status_is_up{job=~\"$job\",instance=~\"$node\"} == 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source error: source_id={{source_id}}, source_name={{source_name}} @ {{instance}}","metric":"","query":"source_status_is_up{job=~\"$job\",instance=~\"$node\"} == 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote storage error {{type}}: {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Errors","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":26},"height":null,"hideTimeOverride":false,"id":13,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Local mode","metric":"","query":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distributed mode","metric":"","query":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Query QPS","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of each type of RisingWave components alive.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":34},"height":null,"hideTimeOverride":false,"id":14,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_type}}","metric":"","query":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of active sessions in frontend nodes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":34},"height":null,"hideTimeOverride":false,"id":15,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Active Sessions","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":42},"height":null,"hideTimeOverride":false,"id":16,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The CPU usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":17,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of CPU cores per RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":18,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU Core Number","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"CPU","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":43},"height":null,"hideTimeOverride":false,"id":19,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The memory usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":20,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Memory","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":21,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (instance) + sum(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (instance) + sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage @ {{instance}}","metric":"","query":"sum(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (instance) + sum(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (instance) + sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Usage (Total)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":22,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(actor_memory_usage[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"streaming actor - {{actor_id}}","metric":"","query":"rate(actor_memory_usage[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage meta cache - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage block cache - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage write buffer - {{job}} @ {{instance}}","metric":"","query":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id) group_left(materialized_view_id) table_info) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized_view {{materialized_view_id}}","metric":"","query":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id) group_left(materialized_view_id) table_info) by (materialized_view_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Usage (Detailed)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Executor cache statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":23,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Join - cache miss - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Join - total lookups - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n appendonly - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n appendonly - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lookup executor - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lookup executor - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal join - cache miss - table_id {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal join - total lookups - table_id {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Materialize - cache hit count - table {{table_id}} - actor {{actor_id}}  {{instance}}","metric":"","query":"rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Materialize - total cache count - table {{table_id}} - actor {{actor_id}}  {{instance}}","metric":"","query":"rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":24,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id)) >=0 ","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"join executor cache miss ratio - - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","query":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id)) >=0 ","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0 ","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0 ","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n appendonly cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream lookup cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream temporal join cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialize executor cache miss ratio - table {{table_id}} - actor {{actor_id}}  {{instance}}","metric":"","query":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Miss Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Storage cache statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":25,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"memory cache - {{table_id}} @ {{type}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_sst_store_block_request_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total_meta_miss_count - {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Storage bloom filter statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":26,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_read_req_check_bloom_filter_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter total - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_read_req_check_bloom_filter_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_read_req_positive_but_non_exist_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter false positive  - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_read_req_positive_but_non_exist_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Bloom Filer","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Storage file cache statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":27,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(file_cache_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"file cache {{op}} @ {{instance}}","metric":"","query":"sum(rate(file_cache_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(file_cache_miss{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"file cache miss @ {{instance}}","metric":"","query":"sum(rate(file_cache_miss{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage File Cache","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Memory","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":44},"height":null,"hideTimeOverride":false,"id":28,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Send/Recv throughput per node for streaming exchange","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":29,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Send @ {{instance}}","metric":"","query":"sum(rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Recv @ {{instance}}","metric":"","query":"sum(rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Streming Remote Exchange (Bytes/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The remote storage read/write throughput per node","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":30,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{instance}}","metric":"","query":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{instance}}","metric":"","query":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Remote I/O (Bytes/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"row"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":31,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{query_id}} : {{source_stage_id}}.{{source_task_id}} -> {{target_stage_id}}.{{target_task_id}}","metric":"","query":"batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Exchange Recv (Rows/s)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Network","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":45},"height":null,"hideTimeOverride":false,"id":32,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"\n                    Objects are classified into 3 groups:\n                    - not referenced by versions: these object are being deleted from object store.\n                    - referenced by non-current versions: these objects are stale (not in the latest version), but those old versions may still be in use (e.g. long-running pinning). Thus those objects cannot be deleted at the moment.\n                    - referenced by current version: these objects are in the latest version.\n                    ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":33,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","query":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","query":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","query":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The storage size of each materialized view","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":34,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_materialized_view_stats{metric='materialized_view_total_size',job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{metric}}, mv id - {{table_id}} ","metric":"","query":"storage_materialized_view_stats{metric='materialized_view_total_size',job=~\"$job\",instance=~\"$node\"}/1024","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"\n                    Objects are classified into 3 groups:\n                    - not referenced by versions: these object are being deleted from object store.\n                    - referenced by non-current versions: these objects are stale (not in the latest version), but those old versions may still be in use (e.g. long-running pinning). Thus those objects cannot be deleted at the moment.\n                    - referenced by current version: these objects are in the latest version.\n                    ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":35,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","query":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","query":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","query":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of bytes that have been written by compaction.Flush refers to the process of compacting Memtables to SSTables at Level 0.Compaction refers to the process of compacting SSTables at one level to another level.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":36,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Compaction - {{job}}","metric":"","query":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush - {{job}}","metric":"","query":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Bytes","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The remote storage read/write throughput","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":37,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}}","metric":"","query":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}}","metric":"","query":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Remote I/O (Bytes/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Size statistics for checkpoint","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":38,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{job}}","metric":"","query":"sum by(le, job) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Checkpoint Size","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Storage","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":46},"height":null,"hideTimeOverride":false,"id":39,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":40,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_name}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of bytes read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":41,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum by (source_id)(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_id}}","metric":"","query":"(sum by (source_id)(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":42,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_source_backfill_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","query":"sum(rate(stream_source_backfill_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Backfill Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized executor actor per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":43,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_executor_row_count{executor_identity=~\".*MaterializeExecutor.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) * on(actor_id) group_left(materialized_view_id, table_name) (group(table_info{table_type=~\"MATERIALIZED_VIEW\",job=~\"$job\",instance=~\"$node\"}) by (actor_id, materialized_view_id, table_name))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized view {{table_name}} table_id {{materialized_view_id}}","metric":"","query":"sum(rate(stream_executor_row_count{executor_identity=~\".*MaterializeExecutor.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) * on(actor_id) group_left(materialized_view_id, table_name) (group(table_info{table_type=~\"MATERIALIZED_VIEW\",job=~\"$job\",instance=~\"$node\"}) by (actor_id, materialized_view_id, table_name))) by (materialized_view_id, table_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been read from the backfill operator used by MV on MV","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":44,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_snapshot_read_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Read Snapshot - table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_backfill_snapshot_read_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_upstream_output_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Upstream - table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_backfill_upstream_output_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"We first record the total blocking duration(ns) of output buffer of each actor. It shows how much time it takes an actor to process a message, i.e. a barrier, a watermark or rows of data, on average. Then we divide this duration by 1 second and show it as a percentage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":45,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, downstream_fragment_id) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}->{{downstream_fragment_id}}","metric":"","query":"avg(rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, downstream_fragment_id) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Output Blocking Time Ratio (Backpressure)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":47},"height":null,"hideTimeOverride":false,"id":46,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":47,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of running query in distributed execution mode","metric":"","query":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Running query in distributed execution mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":48,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of rejected query in distributed execution mode","metric":"","query":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Rejected query in distributed execution mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":49,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of completed query in distributed execution mode","metric":"","query":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Completed query in distributed execution mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":50,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency in Distributed Execution Mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":51,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency in Local Execution Mode","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Batch","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":52,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":53,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(connector_source_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_type}} @ {{source_id}}","metric":"","query":"rate(connector_source_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Source Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":54,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(connector_sink_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector_type}} @ {{sink_id}}","metric":"","query":"rate(connector_sink_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Sink Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Connector Node","transformations":[],"transparent":false,"type":"row"}],"refresh":"","rows":[],"schemaVersion":12,"sharedCrosshair":true,"style":"dark","tags":["risingwave"],"templating":{"list":[{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, instance)","description":"Reporting instance of the metric","hide":0,"includeAll":true,"label":"Node","multi":true,"name":"node","options":[],"query":{"query":"label_values(process_cpu_seconds_total, instance)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, job)","description":"Reporting job of the metric","hide":0,"includeAll":true,"label":"Job","multi":true,"name":"job","options":[],"query":{"query":"label_values(process_cpu_seconds_total, job)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"}]},"time":{"from":"now-30m","to":"now"},"timepicker":{"hidden":false,"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone":"browser","title":"risingwave_dashboard","uid":"Fcy3uV1nz","version":0}
diff --git a/integration_tests/clickhouse-sink/clickhouse_prepare.sql b/integration_tests/clickhouse-sink/clickhouse_prepare.sql
index a30c4c285e45b..1ab869efe39af 100644
--- a/integration_tests/clickhouse-sink/clickhouse_prepare.sql
+++ b/integration_tests/clickhouse-sink/clickhouse_prepare.sql
@@ -30,3 +30,23 @@ CREATE table ck_types (
   c_struct Nested(s_int Int32, s_boolean Bool),
 )ENGINE = ReplacingMergeTree
 PRIMARY KEY (types_id);
+
+CREATE TABLE demo_test_null(
+    user_id Int32,
+    target_id String,
+    event_timestamp DateTime64,
+) ENGINE = Null;
+
+CREATE TABLE demo_test_target_null(
+    user_id Int32,
+    target_id String,
+    event_timestamp DateTime64
+) ENGINE = MergeTree()
+ORDER BY (user_id, event_timestamp);
+
+CREATE MATERIALIZED VIEW demo_mv_null TO demo_test_target_null AS
+SELECT
+    user_id,
+    target_id,
+    event_timestamp
+FROM demo_test_null;
diff --git a/integration_tests/clickhouse-sink/create_sink.sql b/integration_tests/clickhouse-sink/create_sink.sql
index 44163b5a8de31..5f730ed6ff910 100644
--- a/integration_tests/clickhouse-sink/create_sink.sql
+++ b/integration_tests/clickhouse-sink/create_sink.sql
@@ -11,6 +11,19 @@ FROM
     clickhouse.table='demo_test',
 );
 
+CREATE SINK null_clickhouse_sink
+FROM
+    bhv_mv WITH (
+    connector = 'clickhouse',
+    type = 'append-only',
+    force_append_only='true',
+    clickhouse.url = 'http://clickhouse-server-1:8123',
+    clickhouse.user = 'default',
+    clickhouse.password = '',
+    clickhouse.database = 'default',
+    clickhouse.table='demo_test_null',
+);
+
 CREATE SINK ck_types_sink
 FROM
     ck_types WITH (
diff --git a/integration_tests/clickhouse-sink/sink_check.py b/integration_tests/clickhouse-sink/sink_check.py
index 1aa638e2f651d..e51d042d45fd3 100644
--- a/integration_tests/clickhouse-sink/sink_check.py
+++ b/integration_tests/clickhouse-sink/sink_check.py
@@ -2,7 +2,11 @@
 import sys
 
 
-relations = ['default.demo_test', 'default.ck_types']
+relations = [
+    'default.demo_test',
+    'default.demo_test_target_null',
+    'default.ck_types',
+]
 
 failed_cases = []
 for rel in relations:
diff --git a/integration_tests/client-library/java/pom.xml b/integration_tests/client-library/java/pom.xml
index d9383161d1261..66d83eee92651 100644
--- a/integration_tests/client-library/java/pom.xml
+++ b/integration_tests/client-library/java/pom.xml
@@ -19,7 +19,13 @@
     <dependency>
       <groupId>org.postgresql</groupId>
       <artifactId>postgresql</artifactId>
-      <version>42.5.5</version>
+      <version>42.7.3</version>
+    </dependency>
+
+    <dependency>
+      <groupId>commons-dbutils</groupId>
+      <artifactId>commons-dbutils</artifactId>
+      <version>1.8.1</version>
     </dependency>
 
     <!-- JUnit 5 -->
diff --git a/integration_tests/client-library/java/src/test/java/com/risingwave/TestCreateTable.java b/integration_tests/client-library/java/src/test/java/com/risingwave/TestCreateTable.java
index 14bf61ab06595..7694d094120ca 100644
--- a/integration_tests/client-library/java/src/test/java/com/risingwave/TestCreateTable.java
+++ b/integration_tests/client-library/java/src/test/java/com/risingwave/TestCreateTable.java
@@ -8,7 +8,7 @@
 
 public class TestCreateTable {
 
-    public void createSourceTable() throws SQLException {
+    public static void createSourceTable() throws SQLException {
         String createTableQuery;
         Statement statement;
         try (Connection connection = TestUtils.establishConnection()) {
@@ -40,7 +40,7 @@ public void createSourceTable() throws SQLException {
         }
     }
 
-    public void dropSourceTable() throws SQLException {
+    public static void dropSourceTable() throws SQLException {
         String dropSourceQuery = "DROP TABLE s1_java;";
         try (Connection connection = TestUtils.establishConnection()) {
             Statement statement = connection.createStatement();
diff --git a/integration_tests/client-library/java/src/test/java/com/risingwave/TestDatabaseConnection.java b/integration_tests/client-library/java/src/test/java/com/risingwave/TestDatabaseConnection.java
index 30b42b3015808..65a18a8b0f02f 100644
--- a/integration_tests/client-library/java/src/test/java/com/risingwave/TestDatabaseConnection.java
+++ b/integration_tests/client-library/java/src/test/java/com/risingwave/TestDatabaseConnection.java
@@ -23,16 +23,15 @@ static Connection establishConnection() throws SQLException {
 
     @Test
     public void testEstablishConnection() throws SQLException {
-        Connection conn = establishConnection();
-        assertNotNull(conn, "Connection should not be null");
-
-        String query = "SELECT 1";
-        Statement statement = conn.createStatement();
-        ResultSet resultSet = statement.executeQuery(query);
-        assertTrue(resultSet.next(), "Expected a result");
-        int resultValue = resultSet.getInt(1);
-        assertEquals(1, resultValue, "Expected result value to be 1");
-
-        conn.close(); // Close the connection to release resources
+        try (Connection conn = TestUtils.establishConnection()) {
+            assertNotNull(conn, "Connection should not be null");
+
+            String query = "SELECT 1";
+            Statement statement = conn.createStatement();
+            ResultSet resultSet = statement.executeQuery(query);
+            assertTrue(resultSet.next(), "Expected a result");
+            int resultValue = resultSet.getInt(1);
+            assertEquals(1, resultValue, "Expected result value to be 1");
+        }
     }
 }
diff --git a/integration_tests/client-library/java/src/test/java/com/risingwave/TestMaterializedView.java b/integration_tests/client-library/java/src/test/java/com/risingwave/TestMaterializedView.java
index 2d3d537493129..9bcb0207f62aa 100644
--- a/integration_tests/client-library/java/src/test/java/com/risingwave/TestMaterializedView.java
+++ b/integration_tests/client-library/java/src/test/java/com/risingwave/TestMaterializedView.java
@@ -1,11 +1,16 @@
 package com.risingwave;
 
+import org.apache.commons.dbutils.QueryRunner;
+import org.apache.commons.dbutils.handlers.MapListHandler;
+import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
 
 import java.sql.Connection;
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.sql.Statement;
+import java.util.List;
+import java.util.Map;
 
 public class TestMaterializedView {
 
@@ -17,11 +22,38 @@ public void clearDatabase() throws SQLException {
                 statement.executeUpdate(dropViewQuery);
                 System.out.println("Materialized view dropped successfully.");
             }
-            String truncateTableQuery = "DROP TABLE my_table_java;";
+            String truncateTableQuery = "DROP TABLE IF EXISTS my_table_java;";
             try (Statement statement = connection.createStatement()) {
                 statement.executeUpdate(truncateTableQuery);
-                System.out.println("Table dropped successfully.");
+                System.out.println("Table my_table_java dropped successfully.");
             }
+            truncateTableQuery = "DROP TABLE IF EXISTS test_struct;";
+            try (Statement statement = connection.createStatement()) {
+                statement.executeUpdate(truncateTableQuery);
+                System.out.println("Table test_struct dropped successfully.");
+            }
+        }
+    }
+
+    @Test
+    public void testStruct() throws SQLException {
+        try (Connection conn = TestUtils.establishConnection()) {
+            Statement statement = conn.createStatement();
+            statement.executeUpdate("CREATE TABLE IF NOT EXISTS test_struct(" +
+                    "i1 int [], v1 struct<v2 int, v3 double>, t1 timestamptz, c1 varchar" +
+                    ")");
+
+            String insertDataSQL = "INSERT INTO test_struct (i1, v1, t1, c1) VALUES ('{1}', (2, 3), '2020-01-01 01:02:03', 'abc')";
+            statement.execute(insertDataSQL);
+            statement.execute("FLUSH;");
+
+            QueryRunner runner = new QueryRunner();
+            String query = "SELECT * FROM test_struct";
+            List<Map<String, Object>> resultList = runner.query(conn, query, new MapListHandler());
+            Assertions.assertEquals(resultList.size(), 1);
+            Assertions.assertEquals(resultList.get(0).get("v1"), "(2,3)");
+        } finally {
+            clearDatabase();
         }
     }
 
diff --git a/integration_tests/client-library/java/src/test/java/com/risingwave/TestMetadata.java b/integration_tests/client-library/java/src/test/java/com/risingwave/TestMetadata.java
new file mode 100644
index 0000000000000..b076da30d233a
--- /dev/null
+++ b/integration_tests/client-library/java/src/test/java/com/risingwave/TestMetadata.java
@@ -0,0 +1,238 @@
+package com.risingwave;
+
+import org.junit.jupiter.api.Test;
+
+import java.sql.Connection;
+import java.sql.DatabaseMetaData;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+public class TestMetadata {
+
+    public List<String> fetchAllStrings(ResultSet resultSet, String columnName) throws SQLException {
+        List<String> stringColumn = new ArrayList<>();
+
+        // Iterate through the ResultSet and collect schema names
+        while (resultSet.next()) {
+            String element = resultSet.getString(columnName);
+            stringColumn.add(element);
+        }
+
+        return stringColumn;
+    }
+
+    @Test
+    public void testGetSchemas() throws SQLException {
+        try (Connection connection = TestUtils.establishConnection()) {
+            DatabaseMetaData metaData = connection.getMetaData();
+            try (ResultSet schemas = metaData.getSchemas()) {
+                // Check that the ResultSet is not null
+                assertNotNull(schemas);
+
+                List<String> schemaList = fetchAllStrings(schemas, "TABLE_SCHEM");
+
+                // Check that the schema list is not empty
+                assertFalse(schemaList.isEmpty());
+
+                // Check that "pg_catalog" schema exists in the list
+                assertTrue(schemaList.contains("pg_catalog"));
+            }
+        }
+    }
+
+    @Test
+    public void testGetCatalogs() throws SQLException {
+        try (Connection connection = TestUtils.establishConnection()) {
+            DatabaseMetaData metaData = connection.getMetaData();
+            try (ResultSet catalogs = metaData.getCatalogs()) {
+                // Check that the ResultSet is not null
+                assertNotNull(catalogs);
+
+                List<String> catalogList = fetchAllStrings(catalogs, "TABLE_CAT");
+
+                // Check that the catalog list is not empty
+                assertFalse(catalogList.isEmpty());
+
+                // Check that a must-have database "dev" exists in the list
+                assertTrue(catalogList.contains("dev"));
+            }
+        }
+    }
+
+    /*
+     TODO: Support the parameter 'default_transaction_isolation'.
+     @Test
+     public void testGetDefaultTransactionIsolation() throws SQLException {
+         try (Connection connection = TestUtils.establishConnection()) {
+             DatabaseMetaData metaData = connection.getMetaData();
+             metaData.getDefaultTransactionIsolation();
+         }
+     }
+    */
+
+    /*
+    TODO: Support the parameter 'max_index_keys'.
+    @Test
+    public void testGetMaxColumnsInIndex() throws SQLException {
+        try (Connection connection = TestUtils.establishConnection()) {
+            DatabaseMetaData metaData = connection.getMetaData();
+            metaData.getMaxColumnsInIndex();
+            metaData.getImportedKeys("dev", "information_schema", "tables");
+            metaData.getExportedKeys("dev", "information_schema", "tables");
+            metaData.getCrossReference("dev", "information_schema", "tables", "dev", "pg_catalog", "pg_tables");
+        }
+    }
+    */
+
+    /*
+    TODO: Support the `name` type and give it a type length.
+    @Test
+    public void testGetMaxNameLength() throws SQLException {
+        try (Connection connection = TestUtils.establishConnection()) {
+            DatabaseMetaData metaData = connection.getMetaData();
+            metaData.getMaxCursorNameLength();
+            metaData.getMaxSchemaNameLength();
+            metaData.getMaxTableNameLength();
+            metaData.getMaxSchemaNameLength();
+            metaData.getMaxUserNameLength();
+            metaData.getMaxProcedureNameLength();
+            metaData.getMaxCatalogNameLength();
+            metaData.getClientInfoProperties();
+        }
+    }
+    */
+
+    @Test
+    public void testGetProcedures() throws SQLException {
+        try (Connection connection = TestUtils.establishConnection()) {
+            DatabaseMetaData metaData = connection.getMetaData();
+            try (ResultSet procedures = metaData.getProcedures("dev", "public", "")) {
+                assertNotNull(procedures);
+                List<String> procedureList = fetchAllStrings(procedures, "PROCEDURE_NAME");
+                assertTrue(procedureList.isEmpty());
+            }
+        }
+    }
+
+    @Test
+    public void testGetTables() throws SQLException {
+        try (Connection connection = TestUtils.establishConnection()) {
+            DatabaseMetaData metaData = connection.getMetaData();
+            try (ResultSet tables = metaData.getTables("dev", "pg_catalog", "", null)) {
+                assertNotNull(tables);
+                List<String> tablesList = fetchAllStrings(tables, "TABLE_NAME");
+                assertFalse(tablesList.isEmpty());
+            }
+
+            /*
+            TODO: `relacl` is missing in `pg_class`.
+            try (ResultSet acl = metaData.getTablePrivileges("dev", "information_schema", "tables")) {
+                assertNotNull(acl);
+            }
+             */
+        }
+    }
+
+    @Test
+    public void testGetColumns() throws SQLException {
+        try (Connection connection = TestUtils.establishConnection()) {
+            DatabaseMetaData metaData = connection.getMetaData();
+            try (ResultSet columns = metaData.getColumns("dev", "information_schema", "tables", "")) {
+                assertNotNull(columns);
+                List<String> columnList = fetchAllStrings(columns, "COLUMN_NAME");
+                assertFalse(columnList.isEmpty());
+            }
+
+            /*
+            TODO: `relacl` is missing in `pg_class`.
+            try (ResultSet acl = metaData.getColumnPrivileges("dev", "information_schema", "tables", "")) {
+                assertNotNull(acl);
+            }
+             */
+        }
+    }
+
+    /*
+    TODO: the query used in getPrimaryKeys retrieves nothing in RisingWave.
+    @Test
+    public void testGetPrimaryKeys() throws SQLException {
+        try (Connection connection = TestUtils.establishConnection()) {
+            DatabaseMetaData metaData = connection.getMetaData();
+
+            TestUtils.executeDdl(connection, "CREATE TABLE t_test_get_pks (id INT PRIMARY KEY, value TEXT);");
+
+            try (ResultSet pKeys = metaData.getPrimaryKeys("dev", "public", "t_test_get_pks")) {
+                assertNotNull(pKeys);
+                List<String> pKeyList = fetchAllStrings(pKeys, "PK_NAME");
+                assertFalse(pKeyList.isEmpty());
+            }
+
+            try (ResultSet pKeys = metaData.getPrimaryUniqueKeys("dev", "public", "t_test_get_pks")) {
+                assertNotNull(pKeys);
+                List<String> pKeyList = fetchAllStrings(pKeys, "PK_NAME");
+                assertFalse(pKeyList.isEmpty());
+            }
+
+            TestUtils.executeDdl(connection, "DROP TABLE t_test_get_pks;");
+        }
+    }
+    */
+
+
+    /*
+    TODO: Support pg_get_function_result.
+    TODO: prokind is missing in pg_proc.
+    @Test
+    public void testGetFunctions() throws SQLException {
+        try (Connection connection = TestUtils.establishConnection()) {
+            DatabaseMetaData metaData = connection.getMetaData();
+
+            try (ResultSet functions = metaData.getFunctions("dev", "public", "")) {
+                assertNotNull(functions);
+                List<String> fList = fetchAllStrings(functions, "FUNCTION_NAME");
+                assertFalse(fList.isEmpty());
+            }
+
+            try (ResultSet cols = metaData.getFunctionColumns("dev", "public", "")) {
+                assertNotNull(cols);
+            }
+        }
+    }
+    */
+
+    @Test
+    public void testGetUDTs() throws SQLException {
+        try (Connection connection = TestUtils.establishConnection()) {
+            DatabaseMetaData metaData = connection.getMetaData();
+
+            try (ResultSet types = metaData.getUDTs("dev", "public", "", null)) {
+                assertNotNull(types);
+            }
+        }
+    }
+
+    /**
+     * Only checks the API callability, not results.
+     */
+    @Test
+    public void testApiCallable() throws SQLException {
+        try (Connection connection = TestUtils.establishConnection()) {
+            DatabaseMetaData metaData = connection.getMetaData();
+
+            metaData.getTableTypes();
+            metaData.getTypeInfo();
+
+            /// metaData.getIndexInfo();
+
+            // TODO: Actually implement `pg_catalog.pg_get_keywords()`. Now it returns empty.
+            metaData.getSQLKeywords();
+
+            metaData.getBestRowIdentifier("dev", "information_schema", "tables", 1, true);
+            metaData.getVersionColumns("dev", "information_schema", "tables");
+        }
+    }
+}
diff --git a/integration_tests/client-library/java/src/test/java/com/risingwave/TestSubscription.java b/integration_tests/client-library/java/src/test/java/com/risingwave/TestSubscription.java
new file mode 100644
index 0000000000000..0c834e23806b2
--- /dev/null
+++ b/integration_tests/client-library/java/src/test/java/com/risingwave/TestSubscription.java
@@ -0,0 +1,57 @@
+package com.risingwave;
+
+import org.junit.jupiter.api.Test;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+public class TestSubscription {
+    public void createAndTest() throws SQLException {
+        TestCreateTable.createSourceTable();
+        try (Connection conn = TestUtils.establishConnection()) {
+            // Step 1: Create a subscription
+            String createSubscription = "CREATE SUBSCRIPTION sub4 FROM s1_java WITH (retention = '1 hour')";
+            try (PreparedStatement createStmt = conn.prepareStatement(createSubscription)) {
+                createStmt.execute();
+                System.out.println("Subscription created successfully.");
+            }
+            // Step 2: Declare a subscription cursor
+            String declareCursor = "DECLARE cur4 SUBSCRIPTION CURSOR FOR sub4";
+            try (PreparedStatement declareStmt = conn.prepareStatement(declareCursor)) {
+                declareStmt.execute();
+                System.out.println("Subscription cursor declared successfully.");
+            }
+            // Step 3: Fetch data from the subscription cursor
+            String fetchData = "FETCH NEXT FROM cur4";
+            try (PreparedStatement fetchStmt = conn.prepareStatement(fetchData)) {
+                ResultSet rs = fetchStmt.executeQuery();
+                while (rs.next()) {
+                    Object v1 = rs.getObject("v1");
+                    assertNotNull(v1);
+                }
+            }
+        }
+    }
+
+    public void dropSubscription() throws SQLException {
+        try (Connection conn = TestUtils.establishConnection()) {
+            PreparedStatement stmt = conn.prepareStatement("DROP SUBSCRIPTION sub4");
+            stmt.execute();
+            System.out.println("Subscription dropped successfully.");
+        }
+    }
+
+    @Test
+    public void testSubscription() throws SQLException {
+        try {
+            createAndTest();
+        } finally {
+            dropSubscription();
+            TestCreateTable.dropSourceTable();
+        }
+    }
+}
\ No newline at end of file
diff --git a/integration_tests/client-library/java/src/test/java/com/risingwave/TestUtils.java b/integration_tests/client-library/java/src/test/java/com/risingwave/TestUtils.java
index 756da4eb3eb70..245c2f36a06d2 100644
--- a/integration_tests/client-library/java/src/test/java/com/risingwave/TestUtils.java
+++ b/integration_tests/client-library/java/src/test/java/com/risingwave/TestUtils.java
@@ -7,7 +7,8 @@
 
 public class TestUtils {
     public static Connection establishConnection() throws SQLException {
-        final String url = "jdbc:postgresql://risingwave-standalone:4566/dev";
+        // TODO: remove preferQueryMode=simple.
+        final String url = "jdbc:postgresql://risingwave-standalone:4566/dev?preferQueryMode=simple";
         final String user = "root";
         final String password = "";
 
diff --git a/integration_tests/doris-sink/create_sink.sql b/integration_tests/doris-sink/create_sink.sql
index 7cd1ac24857e9..d4702219fed09 100644
--- a/integration_tests/doris-sink/create_sink.sql
+++ b/integration_tests/doris-sink/create_sink.sql
@@ -1,3 +1,5 @@
+create secret doris_secret with (backend = 'meta') as '123456';
+
 CREATE SINK bhv_doris_sink
 FROM
     bhv_mv WITH (
@@ -5,7 +7,7 @@ FROM
     type = 'append-only',
     doris.url = 'http://fe:8030',
     doris.user = 'users',
-    doris.password = '123456',
+    doris.password = secret doris_secret,
     doris.database = 'demo',
     doris.table='demo_bhv_table',
     force_append_only='true'
@@ -18,7 +20,7 @@ FROM
     type = 'upsert',
     doris.url = 'http://fe:8030',
     doris.user = 'users',
-    doris.password = '123456',
+    doris.password = secret doris_secret,
     doris.database = 'demo',
     doris.table='upsert_table',
     primary_key = 'user_id'
diff --git a/integration_tests/feature-store/server/Cargo.lock b/integration_tests/feature-store/server/Cargo.lock
index a7a6edd091224..374f3717dc4df 100644
--- a/integration_tests/feature-store/server/Cargo.lock
+++ b/integration_tests/feature-store/server/Cargo.lock
@@ -1132,9 +1132,9 @@ checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d"
 
 [[package]]
 name = "openssl"
-version = "0.10.60"
+version = "0.10.66"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "79a4c6c3a2b158f7f8f2a2fc5a969fa3a068df6fc9dbb4a43845436e3af7c800"
+checksum = "9529f4786b70a3e8c61e11179af17ab6188ad8d0ded78c5529441ed39d4bd9c1"
 dependencies = [
  "bitflags 2.4.1",
  "cfg-if",
@@ -1164,9 +1164,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"
 
 [[package]]
 name = "openssl-sys"
-version = "0.9.96"
+version = "0.9.103"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3812c071ba60da8b5677cc12bcb1d42989a65553772897a7e0355545a819838f"
+checksum = "7f9e8deee91df40a943c71b917e5874b951d32a802526c85721ce3b776c929d6"
 dependencies = [
  "cc",
  "libc",
diff --git a/integration_tests/iceberg-sink2/docker/hive/config.ini b/integration_tests/iceberg-sink2/docker/hive/config.ini
index d644f3c0d46a1..55a03c3f94753 100644
--- a/integration_tests/iceberg-sink2/docker/hive/config.ini
+++ b/integration_tests/iceberg-sink2/docker/hive/config.ini
@@ -10,7 +10,7 @@ type=append-only
 force_append_only = true
 catalog.type = hive
 catalog.uri = thrift://metastore:9083
-warehouse.path = s3://icebergdata/demo
+warehouse.path = s3a://icebergdata/demo
 s3.endpoint=http://minio-0:9301
 s3.access.key = hummockadmin
 s3.secret.key = hummockadmin
diff --git a/integration_tests/iceberg-sink2/docker/storage/config.ini b/integration_tests/iceberg-sink2/docker/storage/config.ini
index 13e912b8fc3b8..b4e8263f23092 100644
--- a/integration_tests/iceberg-sink2/docker/storage/config.ini
+++ b/integration_tests/iceberg-sink2/docker/storage/config.ini
@@ -14,6 +14,6 @@ s3.secret.key = hummockadmin
 s3.region = ap-southeast-1
 catalog.type = storage
 catalog.name = demo
-warehouse.path = s3://icebergdata/demo
+warehouse.path = s3a://icebergdata/demo
 database.name=s1
 table.name=t1
\ No newline at end of file
diff --git a/integration_tests/iceberg-source/docker/hive/config.ini b/integration_tests/iceberg-source/docker/hive/config.ini
index df07c75258257..80ce27ab14fdd 100644
--- a/integration_tests/iceberg-source/docker/hive/config.ini
+++ b/integration_tests/iceberg-source/docker/hive/config.ini
@@ -8,7 +8,7 @@ port=4566
 connector = iceberg
 catalog.type = hive
 catalog.uri = thrift://metastore:9083
-warehouse.path = s3://icebergdata/demo
+warehouse.path = s3a://icebergdata/demo
 s3.endpoint=http://minio-0:9301
 s3.access.key = hummockadmin
 s3.secret.key = hummockadmin
diff --git a/integration_tests/iceberg-source/docker/storage/config.ini b/integration_tests/iceberg-source/docker/storage/config.ini
index dd795fd3ef684..6439453dcaae6 100644
--- a/integration_tests/iceberg-source/docker/storage/config.ini
+++ b/integration_tests/iceberg-source/docker/storage/config.ini
@@ -11,6 +11,6 @@ s3.access.key = hummockadmin
 s3.secret.key = hummockadmin
 s3.region = ap-southeast-1
 catalog.type = storage
-warehouse.path = s3://icebergdata/demo
+warehouse.path = s3a://icebergdata/demo
 database.name=s1
 table.name=t1
\ No newline at end of file
diff --git a/integration_tests/mysql-cdc/create_source.sql b/integration_tests/mysql-cdc/create_source.sql
index 3f5480ddaa219..5cb99bf4018a8 100644
--- a/integration_tests/mysql-cdc/create_source.sql
+++ b/integration_tests/mysql-cdc/create_source.sql
@@ -1,10 +1,11 @@
+create secret mysql_pwd with (backend = 'meta') as '123456';
 
 create source mysql_mydb with (
     connector = 'mysql-cdc',
     hostname = 'mysql',
     port = '3306',
     username = 'root',
-    password = '123456',
+    password = secret mysql_pwd,
     database.name = 'mydb',
     server.id = '2'
 );
@@ -27,4 +28,4 @@ CREATE TABLE lineitem_rw (
    L_SHIPMODE VARCHAR,
    L_COMMENT VARCHAR,
    PRIMARY KEY(L_ORDERKEY, L_LINENUMBER)
-) FROM mysql_mydb TABLE 'mydb.lineitem';
\ No newline at end of file
+) FROM mysql_mydb TABLE 'mydb.lineitem';
diff --git a/integration_tests/postgres-cdc/create_source.sql b/integration_tests/postgres-cdc/create_source.sql
index 7b24f5da75f34..b7c09694ed09d 100644
--- a/integration_tests/postgres-cdc/create_source.sql
+++ b/integration_tests/postgres-cdc/create_source.sql
@@ -1,3 +1,5 @@
+create secret postgres_pwd with (backend = 'meta') as '123456';
+
 create table person (
     "id" int,
     "name" varchar,
@@ -10,7 +12,7 @@ create table person (
     hostname = 'postgres',
     port = '5432',
     username = 'myuser',
-    password = '123456',
+    password = secret postgres_pwd,
     database.name = 'mydb',
     schema.name = 'public',
     table.name = 'person',
@@ -56,9 +58,9 @@ CREATE TABLE orders_rw (
     hostname = 'postgres',
     port = '5432',
     username = 'myuser',
-    password = '123456',
+    password = secret postgres_pwd,
     database.name = 'mydb',
     schema.name = 'public',
     table.name = 'orders',
     slot.name = 'orders'
-);
\ No newline at end of file
+);
diff --git a/integration_tests/snowflake-sink/README.md b/integration_tests/snowflake-sink/README.md
index 8d38921985b5d..de55dc8c95b32 100644
--- a/integration_tests/snowflake-sink/README.md
+++ b/integration_tests/snowflake-sink/README.md
@@ -18,22 +18,12 @@ note: the required credentials including the following, i.e.,
 - `snowflake.aws_secret_access_key` (a.k.a. the `AWS_SECRET_KEY` in snowflake stage)
 
 ### 1.2 Snowflake setup
-
-users will then need to setup the snowflake, which includes, i.e.,
-- generate the key-value pair for later authentication
-- create a `role` and grant the appropriate permission
-- setup the credential for the user (e.g., `RSA_PUBLIC_KEY`), and retrieve the `snowflake.rsa_public_key_fp` which will later be used in risingwave
-- create a `table` to store the sink data from risingwave
-- create a `stage` to refer the external s3 bucket, which will be used internally by snowflake to load the corresponding data
-- create a `pipe` to actual receive loaded data from the pre-defined stage and copy the data to the snowflake table.
-
-ps.
-1. this assumes the users have already created their accounts and the corresponding databases in snowflake.
-2. for detailed authentication process, refer to [official authentication guide](https://docs.snowflake.com/en/developer-guide/sql-api/authenticating).
-3. for detailed commands, refer to [official reference](https://docs.snowflake.com/en/reference)
+You need to have a table ,a stage and a pipe. In the meantime you need to open s3's SQS queue
+You can complete the above setup by https://docs.snowflake.com/en/user-guide/data-load-snowpipe-auto-s3
 
 an example for snowflake setup commands could be checked at `snowflake_prep.sql`, this also corresponds to the following example sinking use case.
 
+
 ## 2. Begin to sink data
 
 launch your risingwave cluster, and execute the following sql commands respectively.
diff --git a/integration_tests/snowflake-sink/create_sink.sql b/integration_tests/snowflake-sink/create_sink.sql
index 0986005d8c717..6ef762b6708ca 100644
--- a/integration_tests/snowflake-sink/create_sink.sql
+++ b/integration_tests/snowflake-sink/create_sink.sql
@@ -1,18 +1,11 @@
 CREATE SINK snowflake_sink FROM ss_mv WITH (
     connector = 'snowflake',
     type = 'append-only',
-    snowflake.database = 'EXAMPLE_DB',
-    snowflake.schema = 'EXAMPLE_SCHEMA',
-    snowflake.pipe = 'EXAMPLE_SNOWFLAKE_PIPE',
-    snowflake.account_identifier = '<ORG_NAME>-<ACCOUNT_NAME>',
-    snowflake.user = 'XZHSEH',
-    snowflake.rsa_public_key_fp = 'EXAMPLE_FP',
-    snowflake.private_key = 'EXAMPLE_PK',
-    snowflake.s3_bucket = 'EXAMPLE_S3_BUCKET',
-    snowflake.aws_access_key_id = 'EXAMPLE_AWS_ID',
-    snowflake.aws_secret_access_key = 'EXAMPLE_SECRET_KEY',
-    snowflake.aws_region = 'EXAMPLE_REGION',
-    snowflake.s3_path = 'EXAMPLE_S3_PATH',
+    s3.bucket_name = 'EXAMPLE_S3_BUCKET',
+    s3.credentials.access = 'EXAMPLE_AWS_ACCESS',
+    s3.credentials.secret = 'EXAMPLE_AWS_SECRET',
+    s3.region_name = 'ap-EXAMPLE_REGION-2',
+    s3.path = 'EXAMPLE_S3_PATH',
     -- depends on your mv setup, note that snowflake sink *only* supports
     -- `append-only` mode at present.
     force_append_only = 'true'
diff --git a/integration_tests/snowflake-sink/snowflake_prep.sql b/integration_tests/snowflake-sink/snowflake_prep.sql
index b684a93b2d299..33b88c799b900 100644
--- a/integration_tests/snowflake-sink/snowflake_prep.sql
+++ b/integration_tests/snowflake-sink/snowflake_prep.sql
@@ -1,20 +1,5 @@
 USE EXAMPLE_DB;
 
-ALTER USER xzhseh SET RSA_PUBLIC_KEY='your local rsa public key';
-
--- set user permission to account admin level
-GRANT ROLE ACCOUNTADMIN TO USER xzhseh;
-
--- you could either retrieve the fp from desc user's info panel,
--- or from the following select stmt.
-DESC USER xzhseh;
--- also fine, see the documentation for details.
-SELECT TRIM(
-    (SELECT "value" FROM TABLE(RESULT_SCAN(LAST_QUERY_ID()))
-        WHERE "property" = 'RSA_PUBLIC_KEY_FP'),
-    'SHA256:'
-);
-
 -- snowflake table, note to keep the same column name(s).
 CREATE OR REPLACE TABLE example_snowflake_sink_table (user_id INT, target_id VARCHAR, event_timestamp TIMESTAMP_TZ);
 
@@ -28,6 +13,7 @@ CREATE OR REPLACE STAGE example_snowflake_stage
 CREATE OR REPLACE PIPE example_snowflake_pipe
     AS COPY INTO example_snowflake_sink_table
     FROM @example_snowflake_stage MATCH_BY_COLUMN_NAME = CASE_INSENSITIVE;
+-- you will get an AWS SQS ARN, please add it to the list of event notifications in s3
 
 -- select from table after sinking from rw
 SELECT * FROM example_snowflake_sink_table WHERE event_timestamp IS NOT NULL;
diff --git a/integration_tests/snowflake-sink/upsert/create_sink.sql b/integration_tests/snowflake-sink/upsert/create_sink.sql
index 0986005d8c717..fe85d8bec1683 100644
--- a/integration_tests/snowflake-sink/upsert/create_sink.sql
+++ b/integration_tests/snowflake-sink/upsert/create_sink.sql
@@ -1,19 +1,9 @@
 CREATE SINK snowflake_sink FROM ss_mv WITH (
     connector = 'snowflake',
     type = 'append-only',
-    snowflake.database = 'EXAMPLE_DB',
-    snowflake.schema = 'EXAMPLE_SCHEMA',
-    snowflake.pipe = 'EXAMPLE_SNOWFLAKE_PIPE',
-    snowflake.account_identifier = '<ORG_NAME>-<ACCOUNT_NAME>',
-    snowflake.user = 'XZHSEH',
-    snowflake.rsa_public_key_fp = 'EXAMPLE_FP',
-    snowflake.private_key = 'EXAMPLE_PK',
-    snowflake.s3_bucket = 'EXAMPLE_S3_BUCKET',
-    snowflake.aws_access_key_id = 'EXAMPLE_AWS_ID',
-    snowflake.aws_secret_access_key = 'EXAMPLE_SECRET_KEY',
-    snowflake.aws_region = 'EXAMPLE_REGION',
-    snowflake.s3_path = 'EXAMPLE_S3_PATH',
-    -- depends on your mv setup, note that snowflake sink *only* supports
-    -- `append-only` mode at present.
-    force_append_only = 'true'
+    s3.bucket_name = 'EXAMPLE_S3_BUCKET',
+    s3.credentials.access = 'EXAMPLE_AWS_ACCESS',
+    s3.credentials.secret = 'EXAMPLE_AWS_SECRET',
+    s3.region_name = 'ap-EXAMPLE_REGION-2',
+    s3.path = 'EXAMPLE_S3_PATH',
 );
\ No newline at end of file
diff --git a/integration_tests/starrocks-sink/create_sink.sql b/integration_tests/starrocks-sink/create_sink.sql
index 0e96e1f20789b..8d7ebf98dfb20 100644
--- a/integration_tests/starrocks-sink/create_sink.sql
+++ b/integration_tests/starrocks-sink/create_sink.sql
@@ -1,3 +1,5 @@
+create secret starrocks_secret with (backend = 'meta') as '123456';
+
 CREATE SINK bhv_starrocks_sink_primary
 FROM
     bhv_mv WITH (
@@ -7,7 +9,7 @@ FROM
     starrocks.mysqlport = '9030',
     starrocks.httpport = '8030',
     starrocks.user = 'users',
-    starrocks.password = '123456',
+    starrocks.password = secret starrocks_secret,
     starrocks.database = 'demo',
     starrocks.table = 'demo_primary_table',
     force_append_only='true'
@@ -22,7 +24,7 @@ FROM
     starrocks.mysqlport = '9030',
     starrocks.httpport = '8030',
     starrocks.user = 'users',
-    starrocks.password = '123456',
+    starrocks.password = secret starrocks_secret,
     starrocks.database = 'demo',
     starrocks.table = 'demo_duplicate_table',
     force_append_only='true'
@@ -37,7 +39,7 @@ FROM
     starrocks.mysqlport = '9030',
     starrocks.httpport = '8030',
     starrocks.user = 'users',
-    starrocks.password = '123456',
+    starrocks.password = secret starrocks_secret,
     starrocks.database = 'demo',
     starrocks.table = 'demo_aggregate_table',
     force_append_only='true'
@@ -52,7 +54,7 @@ FROM
     starrocks.mysqlport = '9030',
     starrocks.httpport = '8030',
     starrocks.user = 'users',
-    starrocks.password = '123456',
+    starrocks.password = secret starrocks_secret,
     starrocks.database = 'demo',
     starrocks.table = 'demo_unique_table',
     force_append_only='true'
@@ -67,7 +69,7 @@ FROM
     starrocks.mysqlport = '9030',
     starrocks.httpport = '8030',
     starrocks.user = 'users',
-    starrocks.password = '123456',
+    starrocks.password = secret starrocks_secret,
     starrocks.database = 'demo',
     starrocks.table = 'upsert_table',
     primary_key = 'user_id'
@@ -82,7 +84,7 @@ FROM
     starrocks.mysqlport = '9030',
     starrocks.httpport = '8030',
     starrocks.user = 'users',
-    starrocks.password = '123456',
+    starrocks.password = secret starrocks_secret,
     starrocks.database = 'demo',
     starrocks.table = 'starrocks_types',
     force_append_only='true'
diff --git a/java/connector-node/connector-api/src/main/java/com/risingwave/connector/api/source/SourceTypeE.java b/java/connector-node/connector-api/src/main/java/com/risingwave/connector/api/source/SourceTypeE.java
index 0c9858ab4fd5d..21a548bcb825f 100644
--- a/java/connector-node/connector-api/src/main/java/com/risingwave/connector/api/source/SourceTypeE.java
+++ b/java/connector-node/connector-api/src/main/java/com/risingwave/connector/api/source/SourceTypeE.java
@@ -21,6 +21,7 @@ public enum SourceTypeE {
     POSTGRES,
     CITUS,
     MONGODB,
+    SQL_SERVER,
     INVALID;
 
     public static SourceTypeE valueOf(ConnectorServiceProto.SourceType type) {
@@ -33,6 +34,8 @@ public static SourceTypeE valueOf(ConnectorServiceProto.SourceType type) {
                 return SourceTypeE.CITUS;
             case MONGODB:
                 return SourceTypeE.MONGODB;
+            case SQL_SERVER:
+                return SourceTypeE.SQL_SERVER;
             default:
                 return SourceTypeE.INVALID;
         }
diff --git a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/SourceValidateHandler.java b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/SourceValidateHandler.java
index 309ab8db7af4e..458ed8a6d7a3f 100644
--- a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/SourceValidateHandler.java
+++ b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/SourceValidateHandler.java
@@ -157,6 +157,14 @@ public static void validateSource(ConnectorServiceProto.ValidateSourceRequest re
                 var validator = new MongoDbValidator(props);
                 validator.validateDbConfig();
                 break;
+            case SQL_SERVER:
+                ensureRequiredProps(props, isCdcSourceJob);
+                ensurePropNotBlank(props, DbzConnectorConfig.SQL_SERVER_SCHEMA_NAME);
+                try (var sqlServerValidator =
+                        new SqlServerValidator(props, tableSchema, isCdcSourceJob)) {
+                    sqlServerValidator.validateAll();
+                }
+                break;
             default:
                 LOG.warn("Unknown source type");
                 throw ValidatorUtils.invalidArgument("Unknown source type");
diff --git a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/DbzConnectorConfig.java b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/DbzConnectorConfig.java
index a5804974fb29c..fb8aa62916f60 100644
--- a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/DbzConnectorConfig.java
+++ b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/DbzConnectorConfig.java
@@ -55,11 +55,15 @@ public class DbzConnectorConfig {
     public static final String PG_PUB_CREATE = "publication.create.enable";
     public static final String PG_SCHEMA_NAME = "schema.name";
 
+    /* Sql Server configs */
+    public static final String SQL_SERVER_SCHEMA_NAME = "schema.name";
+
     /* RisingWave configs */
     private static final String DBZ_CONFIG_FILE = "debezium.properties";
     private static final String MYSQL_CONFIG_FILE = "mysql.properties";
     private static final String POSTGRES_CONFIG_FILE = "postgres.properties";
     private static final String MONGODB_CONFIG_FILE = "mongodb.properties";
+    private static final String SQL_SERVER_CONFIG_FILE = "sql_server.properties";
 
     private static final String DBZ_PROPERTY_PREFIX = "debezium.";
 
@@ -146,21 +150,21 @@ public DbzConnectorConfig(
                 // If cdc backfill enabled, the source only emit incremental changes, so we must
                 // rewind to the given offset and continue binlog reading from there
                 if (null != startOffset && !startOffset.isBlank()) {
-                    mysqlProps.setProperty("snapshot.mode", "schema_only_recovery");
+                    mysqlProps.setProperty("snapshot.mode", "recovery");
                     mysqlProps.setProperty(
                             ConfigurableOffsetBackingStore.OFFSET_STATE_VALUE, startOffset);
                 } else {
                     // read upstream table schemas and emit incremental changes only
-                    mysqlProps.setProperty("snapshot.mode", "schema_only");
+                    mysqlProps.setProperty("snapshot.mode", "no_data");
                 }
             } else {
                 // if snapshot phase is finished and offset is specified, we will continue binlog
                 // reading from the given offset
                 if (snapshotDone && null != startOffset && !startOffset.isBlank()) {
-                    // 'snapshot.mode=schema_only_recovery' must be configured if binlog offset is
+                    // 'snapshot.mode=recovery' must be configured if binlog offset is
                     // specified. It only snapshots the schemas, not the data, and continue binlog
                     // reading from the specified offset
-                    mysqlProps.setProperty("snapshot.mode", "schema_only_recovery");
+                    mysqlProps.setProperty("snapshot.mode", "recovery");
                     mysqlProps.setProperty(
                             ConfigurableOffsetBackingStore.OFFSET_STATE_VALUE, startOffset);
                 }
@@ -180,7 +184,7 @@ public DbzConnectorConfig(
             }
             if (isCdcBackfill) {
                 // skip the initial snapshot for cdc backfill
-                postgresProps.setProperty("snapshot.mode", "never");
+                postgresProps.setProperty("snapshot.mode", "no_data");
 
                 // if startOffset is specified, we should continue
                 // reading changes from the given offset
@@ -193,7 +197,7 @@ public DbzConnectorConfig(
                 // if snapshot phase is finished and offset is specified, we will continue reading
                 // changes from the given offset
                 if (snapshotDone && null != startOffset && !startOffset.isBlank()) {
-                    postgresProps.setProperty("snapshot.mode", "never");
+                    postgresProps.setProperty("snapshot.mode", "no_data");
                     postgresProps.setProperty(
                             ConfigurableOffsetBackingStore.OFFSET_STATE_VALUE, startOffset);
                 }
@@ -224,7 +228,7 @@ public DbzConnectorConfig(
             // if snapshot phase is finished and offset is specified, we will continue reading
             // changes from the given offset
             if (snapshotDone && null != startOffset && !startOffset.isBlank()) {
-                postgresProps.setProperty("snapshot.mode", "never");
+                postgresProps.setProperty("snapshot.mode", "no_data");
                 postgresProps.setProperty(
                         ConfigurableOffsetBackingStore.OFFSET_STATE_VALUE, startOffset);
             }
@@ -235,7 +239,7 @@ public DbzConnectorConfig(
             // if snapshot phase is finished and offset is specified, we will continue reading
             // changes from the given offset
             if (snapshotDone && null != startOffset && !startOffset.isBlank()) {
-                mongodbProps.setProperty("snapshot.mode", "never");
+                mongodbProps.setProperty("snapshot.mode", "no_data");
                 mongodbProps.setProperty(
                         ConfigurableOffsetBackingStore.OFFSET_STATE_VALUE, startOffset);
             }
@@ -249,7 +253,38 @@ public DbzConnectorConfig(
             mongodbProps.setProperty("name", connectorName);
 
             dbzProps.putAll(mongodbProps);
+        } else if (source == SourceTypeE.SQL_SERVER) {
+            var sqlServerProps = initiateDbConfig(SQL_SERVER_CONFIG_FILE, substitutor);
+            // disable snapshot locking at all
+            sqlServerProps.setProperty("snapshot.locking.mode", "none");
 
+            if (isCdcBackfill) {
+                // if startOffset is specified, we should continue
+                // reading changes from the given offset
+                if (null != startOffset && !startOffset.isBlank()) {
+                    // skip the initial snapshot for cdc backfill
+                    sqlServerProps.setProperty("snapshot.mode", "recovery");
+                    sqlServerProps.setProperty(
+                            ConfigurableOffsetBackingStore.OFFSET_STATE_VALUE, startOffset);
+                } else {
+                    sqlServerProps.setProperty("snapshot.mode", "no_data");
+                }
+            } else {
+                // if snapshot phase is finished and offset is specified, we will continue reading
+                // changes from the given offset
+                if (snapshotDone && null != startOffset && !startOffset.isBlank()) {
+                    sqlServerProps.setProperty("snapshot.mode", "recovery");
+                    sqlServerProps.setProperty(
+                            ConfigurableOffsetBackingStore.OFFSET_STATE_VALUE, startOffset);
+                }
+            }
+            dbzProps.putAll(sqlServerProps);
+            if (isCdcSourceJob) {
+                // remove table filtering for the shared Sql Server source, since we
+                // allow user to ingest tables in different schemas
+                LOG.info("Disable table filtering for the shared Sql Server source");
+                dbzProps.remove("table.include.list");
+            }
         } else {
             throw new RuntimeException("unsupported source type: " + source);
         }
diff --git a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/DbzSourceUtils.java b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/DbzSourceUtils.java
index 83c6d59fac921..13ca00261287b 100644
--- a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/DbzSourceUtils.java
+++ b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/DbzSourceUtils.java
@@ -19,6 +19,7 @@
 import java.sql.DriverManager;
 import java.sql.SQLException;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Optional;
 import java.util.concurrent.TimeUnit;
 import javax.management.JMException;
@@ -112,6 +113,9 @@ public static boolean waitForStreamingRunning(
         } else if (sourceType == SourceTypeE.POSTGRES) {
             return waitForStreamingRunningInner(
                     "postgres", dbServerName, waitStreamingStartTimeout);
+        } else if (sourceType == SourceTypeE.SQL_SERVER) {
+            return waitForStreamingRunningInner(
+                    "sql_server", dbServerName, waitStreamingStartTimeout);
         } else {
             LOG.info("Unsupported backfill source, just return true for {}", dbServerName);
             return true;
@@ -162,12 +166,23 @@ private static boolean isStreamingRunning(String connector, String server, Strin
 
     private static ObjectName getStreamingMetricsObjectName(
             String connector, String server, String context) throws MalformedObjectNameException {
-        return new ObjectName(
-                "debezium."
-                        + connector
-                        + ":type=connector-metrics,context="
-                        + context
-                        + ",server="
-                        + server);
+        if (Objects.equals(connector, "sql_server")) {
+            // TODO: fulfill the task id here, by WKX
+            return new ObjectName(
+                    "debezium."
+                            + connector
+                            + ":type=connector-metrics,task=0,context="
+                            + context
+                            + ",server="
+                            + server);
+        } else {
+            return new ObjectName(
+                    "debezium."
+                            + connector
+                            + ":type=connector-metrics,context="
+                            + context
+                            + ",server="
+                            + server);
+        }
     }
 }
diff --git a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/MySqlValidator.java b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/MySqlValidator.java
index d20a18185a74d..8c122f0f365e5 100644
--- a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/MySqlValidator.java
+++ b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/MySqlValidator.java
@@ -63,7 +63,15 @@ public MySqlValidator(
     @Override
     public void validateDbConfig() {
         try {
-            // TODO: check database server version
+            // Check whether MySQL version is less than 8.4,
+            // since MySQL 8.4 introduces some breaking changes:
+            // https://dev.mysql.com/doc/relnotes/mysql/8.4/en/news-8-4-0.html#mysqld-8-4-0-deprecation-removal
+            var major = jdbcConnection.getMetaData().getDatabaseMajorVersion();
+            var minor = jdbcConnection.getMetaData().getDatabaseMinorVersion();
+
+            if ((major > 8) || (major == 8 && minor >= 4)) {
+                throw ValidatorUtils.failedPrecondition("MySQL version should be less than 8.4");
+            }
             validateBinlogConfig();
         } catch (SQLException e) {
             throw ValidatorUtils.internalError(e.getMessage());
diff --git a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/PostgresValidator.java b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/PostgresValidator.java
index 93d4fdee0bcd4..6e50cb0772ae7 100644
--- a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/PostgresValidator.java
+++ b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/PostgresValidator.java
@@ -180,7 +180,7 @@ private void validateTableSchema() throws SQLException {
         // check primary key
         // reference: https://wiki.postgresql.org/wiki/Retrieve_primary_key_columns
         try (var stmt = jdbcConnection.prepareStatement(ValidatorUtils.getSql("postgres.pk"))) {
-            stmt.setString(1, this.schemaName + "." + this.tableName);
+            stmt.setString(1, String.format("\"%s\".\"%s\"", this.schemaName, this.tableName));
             var res = stmt.executeQuery();
             var pkFields = new HashSet<String>();
             while (res.next()) {
@@ -521,7 +521,8 @@ protected void alterPublicationIfNeeded() throws SQLException {
 
         String alterPublicationSql =
                 String.format(
-                        "ALTER PUBLICATION %s ADD TABLE %s", pubName, schemaName + "." + tableName);
+                        "ALTER PUBLICATION %s ADD TABLE %s",
+                        pubName, String.format("\"%s\".\"%s\"", this.schemaName, this.tableName));
         try (var stmt = jdbcConnection.createStatement()) {
             LOG.info("Altered publication with statement: {}", alterPublicationSql);
             stmt.execute(alterPublicationSql);
diff --git a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/SqlServerValidator.java b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/SqlServerValidator.java
new file mode 100644
index 0000000000000..1c0e8176add74
--- /dev/null
+++ b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/SqlServerValidator.java
@@ -0,0 +1,310 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.risingwave.connector.source.common;
+
+import com.risingwave.connector.api.TableSchema;
+import com.risingwave.connector.api.source.SourceTypeE;
+import com.risingwave.proto.Data;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.util.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class SqlServerValidator extends DatabaseValidator implements AutoCloseable {
+    static final Logger LOG = LoggerFactory.getLogger(SqlServerValidator.class);
+
+    private final TableSchema tableSchema;
+
+    private final Connection jdbcConnection;
+
+    private final String user;
+    private final String dbName;
+    private final String schemaName;
+    private final String tableName;
+
+    // Whether the properties to validate is shared by multiple tables.
+    // If true, we will skip validation check for table
+    private final boolean isCdcSourceJob;
+
+    public SqlServerValidator(
+            Map<String, String> userProps, TableSchema tableSchema, boolean isCdcSourceJob)
+            throws SQLException {
+        this.tableSchema = tableSchema;
+
+        var dbHost = userProps.get(DbzConnectorConfig.HOST);
+        var dbPort = userProps.get(DbzConnectorConfig.PORT);
+        var dbName = userProps.get(DbzConnectorConfig.DB_NAME);
+        var user = userProps.get(DbzConnectorConfig.USER);
+        var password = userProps.get(DbzConnectorConfig.PASSWORD);
+
+        var jdbcUrl = ValidatorUtils.getJdbcUrl(SourceTypeE.SQL_SERVER, dbHost, dbPort, dbName);
+        this.jdbcConnection = DriverManager.getConnection(jdbcUrl, user, password);
+
+        this.dbName = dbName;
+        this.user = user;
+        this.schemaName = userProps.get(DbzConnectorConfig.SQL_SERVER_SCHEMA_NAME);
+        this.tableName = userProps.get(DbzConnectorConfig.TABLE_NAME);
+        this.isCdcSourceJob = isCdcSourceJob;
+    }
+
+    @Override
+    public void validateDbConfig() {
+        try (var stmt =
+                jdbcConnection.prepareStatement(
+                        ValidatorUtils.getSql("sqlserver.db.cdc.enabled"))) {
+            // check whether cdc has been enabled
+            var res = stmt.executeQuery();
+            while (res.next()) {
+                if (!res.getString(1).equals(dbName)) {
+                    throw ValidatorUtils.invalidArgument(
+                            "Sql Server's DB_NAME() '"
+                                    + res.getString(1)
+                                    + "' does not match db_name'"
+                                    + dbName
+                                    + "'.");
+                }
+                if (res.getInt(2) != 1) {
+                    throw ValidatorUtils.invalidArgument(
+                            "Sql Server's '"
+                                    + dbName
+                                    + "' has not enabled CDC.\nPlease modify the config your Sql Server with 'EXEC sys.sp_cdc_enable_db'.");
+                }
+            }
+        } catch (SQLException e) {
+            throw ValidatorUtils.internalError(e.getMessage());
+        }
+        if (isCdcSourceJob) {
+            try (var stmt =
+                    jdbcConnection.prepareStatement(
+                            ValidatorUtils.getSql("sqlserver.sql.agent.enabled"))) {
+                // check whether sql server agent is enabled. It's required to run
+                // fn_cdc_get_max_lsn
+                var res = stmt.executeQuery();
+                while (res.next()) {
+                    if (res.wasNull()) {
+                        throw ValidatorUtils.invalidArgument(
+                                "Sql Server's sql server agent is not activated.\nYou can check it by running `SELECT servicename, startup_type_desc, status_desc FROM sys.dm_server_services WHERE servicename LIKE 'SQL Server Agent%'` in Sql Server.");
+                    }
+                }
+            } catch (SQLException e) {
+                throw ValidatorUtils.internalError(e.getMessage());
+            }
+        }
+    }
+
+    @Override
+    public void validateUserPrivilege() {
+        try {
+            validatePrivileges();
+        } catch (SQLException e) {
+            throw ValidatorUtils.internalError(e.getMessage());
+        }
+    }
+
+    @Override
+    public void validateTable() {
+        try {
+            validateTableSchema();
+        } catch (SQLException e) {
+            throw ValidatorUtils.internalError(e.getMessage());
+        }
+    }
+
+    @Override
+    boolean isCdcSourceJob() {
+        return isCdcSourceJob;
+    }
+
+    private void validateTableSchema() throws SQLException {
+        if (isCdcSourceJob) {
+            return;
+        }
+        // check whether table exist
+        try (var stmt = jdbcConnection.prepareStatement(ValidatorUtils.getSql("sqlserver.table"))) {
+            stmt.setString(1, schemaName);
+            stmt.setString(2, tableName);
+            var res = stmt.executeQuery();
+            while (res.next()) {
+                if (res.getInt(1) == 0) {
+                    throw ValidatorUtils.invalidArgument(
+                            String.format(
+                                    "Sql Server table '%s'.'%s' doesn't exist",
+                                    schemaName, tableName));
+                }
+            }
+        }
+
+        // check cdc enabled
+        try (var stmt =
+                jdbcConnection.prepareStatement(
+                        ValidatorUtils.getSql("sqlserver.table.cdc.enabled"))) {
+            stmt.setString(1, schemaName);
+            stmt.setString(2, tableName);
+            var res = stmt.executeQuery();
+            while (res.next()) {
+                if (res.getInt(1) != 1) {
+                    throw ValidatorUtils.invalidArgument(
+                            "Table '"
+                                    + schemaName
+                                    + "."
+                                    + tableName
+                                    + "' has not enabled CDC.\nPlease ensure CDC is enabled.");
+                }
+            }
+        }
+
+        // check primary key
+        try (var stmt = jdbcConnection.prepareStatement(ValidatorUtils.getSql("sqlserver.pk"))) {
+            stmt.setString(1, this.schemaName);
+            stmt.setString(2, this.tableName);
+            var res = stmt.executeQuery();
+            var pkFields = new HashSet<String>();
+            while (res.next()) {
+                var name = res.getString(1);
+                pkFields.add(name);
+            }
+
+            if (!isPrimaryKeyMatch(tableSchema, pkFields)) {
+                throw ValidatorUtils.invalidArgument("Primary key mismatch");
+            }
+        }
+
+        // Check whether source schema match table schema on upstream
+        // All columns defined must exist in upstream database
+        try (var stmt =
+                jdbcConnection.prepareStatement(ValidatorUtils.getSql("sqlserver.table_schema"))) {
+            stmt.setString(1, this.schemaName);
+            stmt.setString(2, this.tableName);
+            var res = stmt.executeQuery();
+
+            // Field names in lower case -> data type
+            Map<String, String> schema = new HashMap<>();
+            while (res.next()) {
+                var field = res.getString(1);
+                var dataType = res.getString(2);
+                schema.put(field.toLowerCase(), dataType);
+            }
+
+            for (var e : tableSchema.getColumnTypes().entrySet()) {
+                // skip validate internal columns
+                if (e.getKey().startsWith(ValidatorUtils.INTERNAL_COLUMN_PREFIX)) {
+                    continue;
+                }
+                var dataType = schema.get(e.getKey().toLowerCase());
+                if (dataType == null) {
+                    throw ValidatorUtils.invalidArgument(
+                            "Column '" + e.getKey() + "' not found in the upstream database");
+                }
+                if (!isDataTypeCompatible(dataType, e.getValue())) {
+                    throw ValidatorUtils.invalidArgument(
+                            "Incompatible data type of column " + e.getKey());
+                }
+            }
+        }
+    }
+
+    private void validatePrivileges() throws SQLException {
+        if (isCdcSourceJob) {
+            return;
+        }
+
+        try (var stmt =
+                jdbcConnection.prepareStatement(ValidatorUtils.getSql("sqlserver.has.perms"))) {
+            stmt.setString(1, this.schemaName);
+            stmt.setString(2, this.tableName);
+            var res = stmt.executeQuery();
+            while (res.next()) {
+                if (res.getInt(1) != 1) {
+                    throw ValidatorUtils.invalidArgument(
+                            "Sql Server user '"
+                                    + user
+                                    + "' must have select privilege on table '"
+                                    + schemaName
+                                    + "."
+                                    + tableName
+                                    + "''s CDC table.");
+                }
+            }
+        }
+    }
+
+    @Override
+    public void close() throws Exception {
+        if (null != jdbcConnection) {
+            jdbcConnection.close();
+        }
+    }
+
+    public static boolean isPrimaryKeyMatch(TableSchema sourceSchema, Set<String> pkFields) {
+        if (sourceSchema.getPrimaryKeys().size() != pkFields.size()) {
+            return false;
+        }
+        for (var colName : sourceSchema.getPrimaryKeys()) {
+            if (!pkFields.contains(colName)) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    private boolean isDataTypeCompatible(String ssDataType, Data.DataType.TypeName typeName) {
+        // TODO: add more data type compatibility check, by WKX
+        int val = typeName.getNumber();
+        switch (ssDataType) {
+            case "bit":
+                return val == Data.DataType.TypeName.BOOLEAN_VALUE;
+            case "tinyint":
+            case "smallint":
+                return Data.DataType.TypeName.INT16_VALUE <= val
+                        && val <= Data.DataType.TypeName.INT64_VALUE;
+            case "integer":
+                return Data.DataType.TypeName.INT32_VALUE <= val
+                        && val <= Data.DataType.TypeName.INT64_VALUE;
+            case "bigint":
+                return val == Data.DataType.TypeName.INT64_VALUE;
+            case "float":
+            case "real":
+                return val == Data.DataType.TypeName.FLOAT_VALUE
+                        || val == Data.DataType.TypeName.DOUBLE_VALUE;
+            case "boolean":
+                return val == Data.DataType.TypeName.BOOLEAN_VALUE;
+            case "double":
+            case "double precision":
+                return val == Data.DataType.TypeName.DOUBLE_VALUE;
+            case "decimal":
+            case "numeric":
+                return val == Data.DataType.TypeName.DECIMAL_VALUE;
+            case "varchar":
+            case "character varying":
+                return val == Data.DataType.TypeName.VARCHAR_VALUE;
+            case "varbinary":
+                return val == Data.DataType.TypeName.BYTEA_VALUE;
+            case "date":
+                return val == Data.DataType.TypeName.DATE_VALUE;
+            case "time":
+                return val == Data.DataType.TypeName.TIME_VALUE;
+            case "datetime":
+            case "datetime2":
+            case "smalldatetime":
+                return val == Data.DataType.TypeName.TIMESTAMP_VALUE;
+            case "datetimeoffset":
+                return val == Data.DataType.TypeName.TIMESTAMPTZ_VALUE;
+            default:
+                return true; // true for other uncovered types
+        }
+    }
+}
diff --git a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/ValidatorUtils.java b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/ValidatorUtils.java
index 20d631a3267c9..4b79280e62daf 100644
--- a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/ValidatorUtils.java
+++ b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/ValidatorUtils.java
@@ -66,6 +66,10 @@ public static String getJdbcUrl(
             case POSTGRES:
             case CITUS:
                 return String.format("jdbc:postgresql://%s:%s/%s", host, port, database);
+            case SQL_SERVER:
+                return String.format(
+                        "jdbc:sqlserver://%s:%s;databaseName=%s;encrypt=false",
+                        host, port, database);
             default:
                 throw ValidatorUtils.invalidArgument("Unknown source type: " + sourceType);
         }
diff --git a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/core/DbzCdcEngine.java b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/core/DbzCdcEngine.java
index ed22fe36416ec..9227d8225a65c 100644
--- a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/core/DbzCdcEngine.java
+++ b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/core/DbzCdcEngine.java
@@ -47,6 +47,7 @@ public DbzCdcEngine(
                         sourceId,
                         heartbeatTopicPrefix,
                         transactionTopic,
+                        topicPrefix,
                         new ArrayBlockingQueue<>(DEFAULT_QUEUE_CAPACITY));
 
         // Builds a debezium engine but not start it
diff --git a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/core/DbzChangeEventConsumer.java b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/core/DbzChangeEventConsumer.java
index 98a0a171ec4cc..375b4d4a3ad62 100644
--- a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/core/DbzChangeEventConsumer.java
+++ b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/core/DbzChangeEventConsumer.java
@@ -17,6 +17,7 @@
 import com.risingwave.connector.api.source.SourceTypeE;
 import com.risingwave.connector.cdc.debezium.internal.DebeziumOffset;
 import com.risingwave.connector.cdc.debezium.internal.DebeziumOffsetSerializer;
+import com.risingwave.connector.source.common.CdcConnectorException;
 import com.risingwave.proto.ConnectorServiceProto.CdcMessage;
 import com.risingwave.proto.ConnectorServiceProto.GetEventStreamResponse;
 import io.debezium.connector.postgresql.PostgresOffsetContext;
@@ -43,6 +44,7 @@ enum EventType {
     HEARTBEAT,
     TRANSACTION,
     DATA,
+    SCHEMA_CHANGE,
 }
 
 public class DbzChangeEventConsumer
@@ -57,6 +59,7 @@ public class DbzChangeEventConsumer
     private final JsonConverter keyConverter;
     private final String heartbeatTopicPrefix;
     private final String transactionTopic;
+    private final String schemaChangeTopic;
 
     private volatile DebeziumEngine.RecordCommitter<ChangeEvent<SourceRecord, SourceRecord>>
             currentRecordCommitter;
@@ -66,12 +69,14 @@ public class DbzChangeEventConsumer
             long sourceId,
             String heartbeatTopicPrefix,
             String transactionTopic,
+            String schemaChangeTopic,
             BlockingQueue<GetEventStreamResponse> queue) {
         this.connector = connector;
         this.sourceId = sourceId;
         this.outputChannel = queue;
         this.heartbeatTopicPrefix = heartbeatTopicPrefix;
         this.transactionTopic = transactionTopic;
+        this.schemaChangeTopic = schemaChangeTopic;
         LOG.info("heartbeat topic: {}, trnx topic: {}", heartbeatTopicPrefix, transactionTopic);
 
         // The default JSON converter will output the schema field in the JSON which is unnecessary
@@ -105,6 +110,8 @@ private EventType getEventType(SourceRecord record) {
             return EventType.HEARTBEAT;
         } else if (isTransactionMetaEvent(record)) {
             return EventType.TRANSACTION;
+        } else if (isSchemaChangeEvent(record)) {
+            return EventType.SCHEMA_CHANGE;
         } else {
             return EventType.DATA;
         }
@@ -122,6 +129,11 @@ private boolean isTransactionMetaEvent(SourceRecord record) {
         return topic != null && topic.equals(transactionTopic);
     }
 
+    private boolean isSchemaChangeEvent(SourceRecord record) {
+        String topic = record.topic();
+        return topic != null && topic.equals(schemaChangeTopic);
+    }
+
     @Override
     public void handleBatch(
             List<ChangeEvent<SourceRecord, SourceRecord>> events,
@@ -155,7 +167,8 @@ var record = event.value();
             switch (eventType) {
                 case HEARTBEAT:
                     {
-                        var message = msgBuilder.build();
+                        var message =
+                                msgBuilder.setMsgType(CdcMessage.CdcMessageType.HEARTBEAT).build();
                         LOG.debug("heartbeat => {}", message.getOffset());
                         respBuilder.addEvents(message);
                         break;
@@ -168,7 +181,7 @@ var record = event.value();
                                         record.topic(), record.valueSchema(), record.value());
                         var message =
                                 msgBuilder
-                                        .setIsTransactionMeta(true)
+                                        .setMsgType(CdcMessage.CdcMessageType.TRANSACTION_META)
                                         .setPayload(new String(payload, StandardCharsets.UTF_8))
                                         .setSourceTsMs(trxTs)
                                         .build();
@@ -176,6 +189,46 @@ var record = event.value();
                         respBuilder.addEvents(message);
                         break;
                     }
+
+                case SCHEMA_CHANGE:
+                    {
+                        var sourceStruct = ((Struct) record.value()).getStruct("source");
+                        if (sourceStruct == null) {
+                            throw new CdcConnectorException(
+                                    "source field is missing in schema change event");
+                        }
+
+                        // upstream event time
+                        long sourceTsMs = sourceStruct.getInt64("ts_ms");
+                        byte[] payload =
+                                payloadConverter.fromConnectData(
+                                        record.topic(), record.valueSchema(), record.value());
+
+                        // We intentionally don't set the fullTableName for schema change event,
+                        // since it doesn't need to be routed to a specific cdc table
+                        var message =
+                                msgBuilder
+                                        .setMsgType(CdcMessage.CdcMessageType.SCHEMA_CHANGE)
+                                        .setPayload(new String(payload, StandardCharsets.UTF_8))
+                                        .setSourceTsMs(sourceTsMs)
+                                        .build();
+                        LOG.debug(
+                                "offset => {}, key => {}, payload => {}",
+                                message.getOffset(),
+                                message.getKey(),
+                                message.getPayload());
+                        respBuilder.addEvents(message);
+
+                        // emit the schema change event as a single response
+                        respBuilder.setSourceId(sourceId);
+                        var response = respBuilder.build();
+                        outputChannel.put(response);
+
+                        // reset the response builder
+                        respBuilder = GetEventStreamResponse.newBuilder();
+                        break;
+                    }
+
                 case DATA:
                     {
                         // Topic naming conventions
@@ -192,10 +245,11 @@ var record = event.value();
                         }
                         // get upstream event time from the "source" field
                         var sourceStruct = ((Struct) record.value()).getStruct("source");
-                        long sourceTsMs =
-                                sourceStruct == null
-                                        ? System.currentTimeMillis()
-                                        : sourceStruct.getInt64("ts_ms");
+                        if (sourceStruct == null) {
+                            throw new CdcConnectorException(
+                                    "source field is missing in data change event");
+                        }
+                        long sourceTsMs = sourceStruct.getInt64("ts_ms");
                         byte[] payload =
                                 payloadConverter.fromConnectData(
                                         record.topic(), record.valueSchema(), record.value());
@@ -208,6 +262,7 @@ var record = event.value();
                         String msgKey = key == null ? "" : new String(key, StandardCharsets.UTF_8);
                         var message =
                                 msgBuilder
+                                        .setMsgType(CdcMessage.CdcMessageType.DATA)
                                         .setFullTableName(fullTableName)
                                         .setPayload(msgPayload)
                                         .setKey(msgKey)
diff --git a/java/connector-node/risingwave-connector-service/src/main/resources/sql_server.properties b/java/connector-node/risingwave-connector-service/src/main/resources/sql_server.properties
new file mode 100644
index 0000000000000..0e0c55c939ef5
--- /dev/null
+++ b/java/connector-node/risingwave-connector-service/src/main/resources/sql_server.properties
@@ -0,0 +1,24 @@
+# configs for sql server conneoctor
+connector.class=io.debezium.connector.sqlserver.SqlServerConnector
+# default snapshot mode to initial
+snapshot.mode=${debezium.snapshot.mode:-initial}
+database.hostname=${hostname}
+database.port=${port}
+database.user=${username}
+database.password=${password}
+database.names=${database.name}
+table.include.list=${schema.name}.${table.name:-*}
+# only read table schema of the captured tables in the specified database
+schema.history.internal.store.only.captured.tables.ddl=true
+schema.history.internal.store.only.captured.databases.ddl=true
+# default to disable schema change events
+include.schema.changes=${debezium.include.schema.changes:-false}
+# default heartbeat interval 5 mins
+heartbeat.interval.ms=${debezium.heartbeat.interval.ms:-300000}
+# In sharing cdc source mode, we will subscribe to multiple tables in the given database,
+# so here we set ${table.name} to a default value `RW_CDC_Sharing` just for display.
+name=${hostname}:${port}:${database.name}.${schema.name}.${table.name:-RW_CDC_Sharing}
+# In sharing cdc mode, transaction metadata will be enabled in frontend.
+# For sql server, it's always false actually.
+provide.transaction.metadata=${transactional:-false}
+database.encrypt=false
diff --git a/java/connector-node/risingwave-connector-service/src/main/resources/validate_sql.properties b/java/connector-node/risingwave-connector-service/src/main/resources/validate_sql.properties
index 769c3cb1c8fb0..54214b6b46190 100644
--- a/java/connector-node/risingwave-connector-service/src/main/resources/validate_sql.properties
+++ b/java/connector-node/risingwave-connector-service/src/main/resources/validate_sql.properties
@@ -37,5 +37,12 @@ GROUP BY r1.rolname \
 ), \
 tmp AS (SELECT DISTINCT(UNNEST(m)) AS members FROM base) \
 SELECT ARRAY_AGG(members) AS members FROM tmp
+sqlserver.db.cdc.enabled=SELECT name, is_cdc_enabled FROM sys.databases WHERE name = DB_NAME()
+sqlserver.table=SELECT count(*) FROM information_schema.TABLES WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ?
+sqlserver.table.cdc.enabled=SELECT COUNT(*) FROM cdc.change_tables AS ct INNER JOIN sys.tables AS t ON ct.source_object_id = t.object_id INNER JOIN sys.schemas AS s ON t.schema_id = s.schema_id WHERE s.name = ? AND t.name = ?
+sqlserver.pk=SELECT k.column_name FROM information_schema.table_constraints t INNER JOIN information_schema.key_column_usage k ON t.constraint_name = k.constraint_name AND t.table_name = k.table_name WHERE t.constraint_type = 'PRIMARY KEY' AND t.table_schema = ? AND t.table_name = ?
+sqlserver.table_schema=SELECT COLUMN_NAME, DATA_TYPE FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ? ORDER BY ORDINAL_POSITION
+sqlserver.has.perms=SELECT HAS_PERMS_BY_NAME('cdc.' + ct.capture_instance + '_CT', 'OBJECT', 'SELECT') FROM cdc.change_tables AS ct INNER JOIN sys.tables AS t ON ct.source_object_id = t.object_id INNER JOIN sys.schemas AS s ON t.schema_id = s.schema_id WHERE s.name = ? AND t.name = ?
+sqlserver.sql.agent.enabled=SELECT sys.fn_cdc_get_max_lsn()
 citus.distributed_table=select citus_table_type from citus_tables where table_name=?::regclass
 postgres.rds.role.check=SELECT r.rolname, r.rolsuper, r.rolinherit, r.rolcreaterole, r.rolcreatedb, r.rolcanlogin, r.rolconnlimit, r.rolvaliduntil, ARRAY(SELECT b.rolname FROM pg_catalog.pg_auth_members m JOIN pg_catalog.pg_roles b ON (m.roleid = b.oid) WHERE m.member = r.oid) as memberof , r.rolreplication , r.rolbypassrls FROM pg_catalog.pg_roles r WHERE r.rolname = ?
diff --git a/java/connector-node/risingwave-sink-cassandra/src/main/java/com/risingwave/connector/CassandraUtil.java b/java/connector-node/risingwave-sink-cassandra/src/main/java/com/risingwave/connector/CassandraUtil.java
index 8327893f6da9a..463bdb9d3f113 100644
--- a/java/connector-node/risingwave-sink-cassandra/src/main/java/com/risingwave/connector/CassandraUtil.java
+++ b/java/connector-node/risingwave-sink-cassandra/src/main/java/com/risingwave/connector/CassandraUtil.java
@@ -91,7 +91,7 @@ public static void checkSchema(
                 throw Status.FAILED_PRECONDITION
                         .withDescription(
                                 String.format(
-                                        "Don't match in the name, rw is %s cassandra can't find it",
+                                        "Name mismatch. Column `%s` on RisingWave side is not found on Cassandra side.",
                                         columnDesc.getName()))
                         .asRuntimeException();
             }
@@ -100,7 +100,7 @@ public static void checkSchema(
                 throw Status.FAILED_PRECONDITION
                         .withDescription(
                                 String.format(
-                                        "Don't match in the type, name is %s, cassandra is %s, rw is %s",
+                                        "Data type mismatch for column `%s`. Cassandra side: `%s`, RisingWave side: `%s`.",
                                         columnDesc.getName(),
                                         cassandraColumnDescMap.get(cql),
                                         columnDesc.getDataType().getTypeName()))
@@ -113,7 +113,7 @@ public static void checkPrimaryKey(
             List<ColumnMetadata> cassandraColumnMetadatas, List<String> columnMetadatas) {
         if (cassandraColumnMetadatas.size() != columnMetadatas.size()) {
             throw Status.FAILED_PRECONDITION
-                    .withDescription("Primary key len don't match")
+                    .withDescription("Primary key length mismatch.")
                     .asRuntimeException();
         }
         Set<String> cassandraColumnsSet =
@@ -125,7 +125,7 @@ public static void checkPrimaryKey(
                 throw Status.FAILED_PRECONDITION
                         .withDescription(
                                 String.format(
-                                        "Primary key don't match. RisingWave Primary key is %s, don't find it in cassandra",
+                                        "Primary key mismatch. Primary key `%s` on RisingWave side is not found on Cassandra side",
                                         columnMetadata))
                         .asRuntimeException();
             }
diff --git a/java/connector-node/risingwave-sink-mock-flink/risingwave-sink-mock-flink-common/src/main/java/com/risingwave/mock/flink/common/FlinkDynamicUtil.java b/java/connector-node/risingwave-sink-mock-flink/risingwave-sink-mock-flink-common/src/main/java/com/risingwave/mock/flink/common/FlinkDynamicUtil.java
index 280b6b7eccb44..4a58c3160b075 100644
--- a/java/connector-node/risingwave-sink-mock-flink/risingwave-sink-mock-flink-common/src/main/java/com/risingwave/mock/flink/common/FlinkDynamicUtil.java
+++ b/java/connector-node/risingwave-sink-mock-flink/risingwave-sink-mock-flink-common/src/main/java/com/risingwave/mock/flink/common/FlinkDynamicUtil.java
@@ -120,14 +120,15 @@ public static void validateSchemaWithCatalog(
                 throw Status.FAILED_PRECONDITION
                         .withDescription(
                                 String.format(
-                                        "Don't match in the name, rw is %s", columnDesc.getName()))
+                                        "Name mismatch. Column `%s` on RisingWave side is not found on Flink side.",
+                                        columnDesc.getName()))
                         .asRuntimeException();
             }
             if (!checkType(columnDesc.getDataType(), flinkColumnMap.get(columnDesc.getName()))) {
                 throw Status.FAILED_PRECONDITION
                         .withDescription(
                                 String.format(
-                                        "Don't match in the type, name is %s, Sink is %s, rw is %s",
+                                        "Data type mismatch for column `%s`. Flink side: `%s`, RisingWave side: `%s`.",
                                         columnDesc.getName(),
                                         flinkColumnMap.get(columnDesc.getName()),
                                         columnDesc.getDataType().getTypeName()))
diff --git a/java/connector-node/risingwave-source-cdc/pom.xml b/java/connector-node/risingwave-source-cdc/pom.xml
index 5ee531ef805e9..fb1519f6a183c 100644
--- a/java/connector-node/risingwave-source-cdc/pom.xml
+++ b/java/connector-node/risingwave-source-cdc/pom.xml
@@ -47,6 +47,10 @@
             <groupId>io.debezium</groupId>
             <artifactId>debezium-connector-mongodb</artifactId>
         </dependency>
+        <dependency>
+            <groupId>io.debezium</groupId>
+            <artifactId>debezium-connector-sqlserver</artifactId>
+        </dependency>
 
         <!-- database dependencies  -->
         <dependency>
@@ -56,7 +60,8 @@
         <dependency>
             <groupId>com.zendesk</groupId>
             <artifactId>mysql-binlog-connector-java</artifactId>
-            <version>0.27.2</version>
+            <!-- https://central.sonatype.com/artifact/com.zendesk/mysql-binlog-connector-java/versions -->
+            <version>0.29.0</version>
         </dependency>
         <dependency>
             <groupId>org.postgresql</groupId>
diff --git a/java/pom.xml b/java/pom.xml
index 644588c9d6b44..c0521bd6dfc4a 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -74,7 +74,7 @@
         <commons.io.version>2.11.0</commons.io.version>
         <commons.text.version>1.10.0</commons.text.version>
         <commons.lang3.version>3.12.0</commons.lang3.version>
-        <debezium.version>2.4.2.Final</debezium.version>
+        <debezium.version>2.6.2.Final</debezium.version>
         <jackson.version>2.15.0</jackson.version>
         <spark_sql.version>3.3.1</spark_sql.version>
         <hadoop.version>3.4.0</hadoop.version>
@@ -173,6 +173,11 @@
                 <artifactId>debezium-connector-mongodb</artifactId>
                 <version>${debezium.version}</version>
             </dependency>
+            <dependency>
+                <groupId>io.debezium</groupId>
+                <artifactId>debezium-connector-sqlserver</artifactId>
+                <version>${debezium.version}</version>
+            </dependency>
             <dependency>
                 <groupId>org.postgresql</groupId>
                 <artifactId>postgresql</artifactId>
diff --git a/lints/Cargo.lock b/lints/Cargo.lock
index 3a9bd8384c0d9..e3b748e6da670 100644
--- a/lints/Cargo.lock
+++ b/lints/Cargo.lock
@@ -162,8 +162,7 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "clippy_config"
-version = "0.1.79"
-source = "git+https://github.com/rust-lang/rust-clippy?rev=fca4e16ffb8c07186ee23becd44cd5c9fb51896c#fca4e16ffb8c07186ee23becd44cd5c9fb51896c"
+version = "0.1.80"
 dependencies = [
  "rustc-semver",
  "serde",
@@ -172,8 +171,7 @@ dependencies = [
 
 [[package]]
 name = "clippy_utils"
-version = "0.1.79"
-source = "git+https://github.com/rust-lang/rust-clippy?rev=fca4e16ffb8c07186ee23becd44cd5c9fb51896c#fca4e16ffb8c07186ee23becd44cd5c9fb51896c"
+version = "0.1.80"
 dependencies = [
  "arrayvec",
  "clippy_config",
diff --git a/lints/Cargo.toml b/lints/Cargo.toml
index 74fc49c3fd080..43ece1f6fc5b7 100644
--- a/lints/Cargo.toml
+++ b/lints/Cargo.toml
@@ -14,7 +14,7 @@ path = "ui/format_error.rs"
 # See `README.md` before bumping the version.
 # Remember to update the version in `ci/Dockerfile` as well.
 [dependencies]
-clippy_utils = { git = "https://github.com/rust-lang/rust-clippy", rev = "fca4e16ffb8c07186ee23becd44cd5c9fb51896c" }
+clippy_utils = { git = "https://github.com/risingwavelabs/clippy", rev = "5e2a7c6adebdb0478ee6d5b67ab4ee94153b2997" }
 dylint_linting = "3.1.0"
 itertools = "0.12"
 
diff --git a/lints/README.md b/lints/README.md
index 5007474227ab3..3ab55f0bbfe7a 100644
--- a/lints/README.md
+++ b/lints/README.md
@@ -30,8 +30,13 @@ Duplicate `.vscode/settings.json.example` to `.vscode/settings.json` to enable r
 
 ## Bump toolchain
 
-The version of the toolchain is specified in `rust-toolchain` file under current directory.
-It does not have to be exactly the same as the one used to build RisingWave, but it should be close enough to avoid compile errors.
+The version of the toolchain is specified in `rust-toolchain` file under current directory. It will be used to build the lints, and also be used by `dylint` to compile RisingWave, instead of the root-level `rust-toolchain`.
+
+So the chosen toolchain needs to
+1. be close enough to the root-level `rust-toolchain` to make RisingWave compile. It does not have to be exactly the same version though.
+2. be close enough to the dependency `clippy_utils`'s corresponding `rust-toolchain` in the Clippy's repo.
+
+(Note: `clippy_utils` depends on rustc's internal unstable API. When rustc has breaking changes, the `rust` repo's Clippy will be updated. And then it's [synced back to the Clippy repo bi-weekly](https://doc.rust-lang.org/clippy/development/infrastructure/sync.html#syncing-changes-between-clippy-and-rust-langrust). So ideally we can use `clippy_utils` in the rust repo corresponding to our root-level nightly version, but that repo is too large. Perhaps we can also consider copy the code out to workaround this problem.)
 
 The information below can be helpful in finding the appropriate version to bump to.
 
diff --git a/lints/rust-toolchain b/lints/rust-toolchain
index 3bbdf2b2d53fd..a146af66cd637 100644
--- a/lints/rust-toolchain
+++ b/lints/rust-toolchain
@@ -1,5 +1,5 @@
 # See `README.md` before bumping the version.
 
 [toolchain]
-channel = "nightly-2024-04-18"
+channel = "nightly-2024-06-06"
 components = ["llvm-tools-preview", "rustc-dev"]
diff --git a/lints/src/format_error.rs b/lints/src/format_error.rs
index 9b6d9be5b8c4e..402adc4aa5af0 100644
--- a/lints/src/format_error.rs
+++ b/lints/src/format_error.rs
@@ -14,7 +14,7 @@
 
 use clippy_utils::diagnostics::span_lint_and_help;
 use clippy_utils::macros::{
-    find_format_arg_expr, find_format_args, is_format_macro, macro_backtrace,
+    find_format_arg_expr, is_format_macro, macro_backtrace, FormatArgsStorage,
 };
 use clippy_utils::ty::{implements_trait, match_type};
 use clippy_utils::{
@@ -56,7 +56,15 @@ declare_tool_lint! {
 }
 
 #[derive(Default)]
-pub struct FormatError;
+pub struct FormatError {
+    format_args: FormatArgsStorage,
+}
+
+impl FormatError {
+    pub fn new(format_args: FormatArgsStorage) -> Self {
+        Self { format_args }
+    }
+}
 
 impl_lint_pass!(FormatError => [FORMAT_ERROR]);
 
@@ -90,7 +98,7 @@ impl<'tcx> LateLintPass<'tcx> for FormatError {
 
         for macro_call in macro_backtrace(expr.span) {
             if is_format_macro(cx, macro_call.def_id)
-                && let Some(format_args) = find_format_args(cx, expr, macro_call.expn)
+                && let Some(format_args) = self.format_args.get(cx, expr, macro_call.expn)
             {
                 for piece in &format_args.template {
                     if let FormatArgsPiece::Placeholder(placeholder) = piece
diff --git a/lints/src/lib.rs b/lints/src/lib.rs
index df77538d3cf17..6928bcd028a8c 100644
--- a/lints/src/lib.rs
+++ b/lints/src/lib.rs
@@ -14,7 +14,6 @@
 
 #![feature(rustc_private)]
 #![feature(let_chains)]
-#![feature(lazy_cell)]
 #![warn(unused_extern_crates)]
 
 extern crate rustc_ast;
@@ -36,13 +35,19 @@ pub fn register_lints(_sess: &rustc_session::Session, lint_store: &mut rustc_lin
     // -- Begin lint registration --
 
     // Preparation steps.
-    lint_store.register_early_pass(|| {
-        Box::<utils::format_args_collector::FormatArgsCollector>::default()
+    let format_args_storage = clippy_utils::macros::FormatArgsStorage::default();
+    let format_args = format_args_storage.clone();
+    lint_store.register_early_pass(move || {
+        Box::new(utils::format_args_collector::FormatArgsCollector::new(
+            format_args.clone(),
+        ))
     });
 
     // Actual lints.
     lint_store.register_lints(&[format_error::FORMAT_ERROR]);
-    lint_store.register_late_pass(|_| Box::<format_error::FormatError>::default());
+    let format_args = format_args_storage.clone();
+    lint_store
+        .register_late_pass(move |_| Box::new(format_error::FormatError::new(format_args.clone())));
 
     // --  End lint registration  --
 
diff --git a/lints/src/utils/format_args_collector.rs b/lints/src/utils/format_args_collector.rs
index 7524169666d97..a3b144a6a8aeb 100644
--- a/lints/src/utils/format_args_collector.rs
+++ b/lints/src/utils/format_args_collector.rs
@@ -12,15 +12,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-//! Copied from `https://github.com/rust-lang/rust-clippy/blob/8b0bf6423dfaf5545014db85fcba7bc745beed4c/clippy_lints/src/utils/format_args_collector.rs`
-//!
-//! Init `AST_FORMAT_ARGS` before running the late pass, so that we can call `find_format_args`.
+//! Copied from `https://github.com/rust-lang/rust-clippy/blob/993d8ae2a7b26ac779fde923b2ce9ce35d7143a8/clippy_lints/src/utils/format_args_collector.rs`
 
 use std::iter::once;
 use std::mem;
-use std::rc::Rc;
 
-use clippy_utils::macros::AST_FORMAT_ARGS;
+use clippy_utils::macros::FormatArgsStorage;
 use clippy_utils::source::snippet_opt;
 use itertools::Itertools;
 use rustc_ast::{Crate, Expr, ExprKind, FormatArgs};
@@ -30,11 +27,19 @@ use rustc_lint::{EarlyContext, EarlyLintPass};
 use rustc_session::impl_lint_pass;
 use rustc_span::{hygiene, Span};
 
-/// Collects [`rustc_ast::FormatArgs`] so that future late passes can call
-/// [`clippy_utils::macros::find_format_args`]
-#[derive(Default)]
+/// Populates [`FormatArgsStorage`] with AST [`FormatArgs`] nodes
 pub struct FormatArgsCollector {
-    format_args: FxHashMap<Span, Rc<FormatArgs>>,
+    format_args: FxHashMap<Span, FormatArgs>,
+    storage: FormatArgsStorage,
+}
+
+impl FormatArgsCollector {
+    pub fn new(storage: FormatArgsStorage) -> Self {
+        Self {
+            format_args: FxHashMap::default(),
+            storage,
+        }
+    }
 }
 
 impl_lint_pass!(FormatArgsCollector => []);
@@ -47,15 +52,12 @@ impl EarlyLintPass for FormatArgsCollector {
             }
 
             self.format_args
-                .insert(expr.span.with_parent(None), Rc::new((**args).clone()));
+                .insert(expr.span.with_parent(None), (**args).clone());
         }
     }
 
     fn check_crate_post(&mut self, _: &EarlyContext<'_>, _: &Crate) {
-        AST_FORMAT_ARGS.with(|ast_format_args| {
-            let result = ast_format_args.set(mem::take(&mut self.format_args));
-            debug_assert!(result.is_ok());
-        });
+        self.storage.set(mem::take(&mut self.format_args));
     }
 }
 
diff --git a/proto/batch_plan.proto b/proto/batch_plan.proto
index 592b60a5ab0ac..9b12d0b583d1e 100644
--- a/proto/batch_plan.proto
+++ b/proto/batch_plan.proto
@@ -28,6 +28,7 @@ message RowSeqScanNode {
 
   // The pushed down `batch_limit`. Max rows needed to return.
   optional uint64 limit = 6;
+  optional plan_common.AsOf as_of = 7;
 }
 
 message SysRowSeqScanNode {
@@ -65,6 +66,26 @@ message SourceNode {
   map<string, secret.SecretRef> secret_refs = 6;
 }
 
+message FileScanNode {
+  enum FileFormat {
+    FILE_FORMAT_UNSPECIFIED = 0;
+    PARQUET = 1;
+  }
+
+  enum StorageType {
+    STORAGE_TYPE_UNSPECIFIED = 0;
+    S3 = 1;
+  }
+
+  repeated plan_common.ColumnDesc columns = 1;
+  FileFormat file_format = 2;
+  StorageType storage_type = 3;
+  string s3_region = 4;
+  string s3_access_key = 5;
+  string s3_secret_key = 6;
+  repeated string file_location = 7;
+}
+
 message ProjectNode {
   repeated expr.ExprNode select_list = 1;
 }
@@ -279,6 +300,7 @@ message LocalLookupJoinNode {
   // Null safe means it treats `null = null` as true.
   // Each key pair can be null safe independently. (left_key, right_key, null_safe)
   repeated bool null_safe = 11;
+  optional plan_common.AsOf as_of = 12;
 }
 
 // RFC: A new schedule way for distributed lookup join
@@ -295,6 +317,7 @@ message DistributedLookupJoinNode {
   // Null safe means it treats `null = null` as true.
   // Each key pair can be null safe independently. (left_key, right_key, null_safe)
   repeated bool null_safe = 9;
+  optional plan_common.AsOf as_of = 10;
 }
 
 message UnionNode {}
@@ -341,6 +364,7 @@ message PlanNode {
     SortOverWindowNode sort_over_window = 35;
     MaxOneRowNode max_one_row = 36;
     LogRowSeqScanNode log_row_seq_scan = 37;
+    FileScanNode file_scan = 38;
     // The following nodes are used for testing.
     bool block_executor = 100;
     bool busy_loop_executor = 101;
diff --git a/proto/catalog.proto b/proto/catalog.proto
index 8a360418159b3..e13a99ac3c2f9 100644
--- a/proto/catalog.proto
+++ b/proto/catalog.proto
@@ -52,6 +52,11 @@ message StreamSourceInfo {
   // deprecated
   plan_common.RowFormatType row_format = 1;
   string row_schema_location = 2;
+  // This means *use **confluent** schema registry* and is `false` for **aws glue** schema registry.
+  // Eventually we will deprecate it and rely on `enum SchemaLocation` derived from `format_encode_options` below.
+  // * schema.location     false
+  // * schema.registry     true
+  // * aws.glue.schema_arn false
   bool use_schema_registry = 3;
   string proto_message_name = 4;
   int32 csv_delimiter = 5;
@@ -72,7 +77,7 @@ message StreamSourceInfo {
   //
   // Currently, the following sources can be shared:
   //
-  // - Direct CDC sources (mysql & postgresql)
+  // - Direct CDC sources (mysql & postgresql & sqlserver)
   // - MQ sources (Kafka)
   bool cdc_source_job = 13;
   // Only used when `cdc_source_job` is `true`.
@@ -130,6 +135,8 @@ message Source {
 
   // Per-source catalog version, used by schema change.
   uint64 version = 100;
+
+  optional uint32 rate_limit = 101;
 }
 
 enum SinkType {
@@ -190,6 +197,9 @@ message Sink {
   // Handle the sink relies on any sceret. The key is the propertity name and the value is the secret id and type.
   // Used for connect options.
   map<string, secret.SecretRef> secret_refs = 25;
+
+  // only for the sink whose target is a table. Columns of the target table when the sink is created. At this point all the default columns of the target table are all handled by the project operator in the sink plan.
+  repeated plan_common.ColumnCatalog original_target_columns = 26;
 }
 
 message Subscription {
@@ -458,3 +468,8 @@ message Secret {
   uint32 owner = 5;
   uint32 schema_id = 6;
 }
+
+message OptionsWithSecret {
+  map<string, string> options = 1;
+  map<string, secret.SecretRef> secret_refs = 2;
+}
diff --git a/proto/common.proto b/proto/common.proto
index 164150379c484..d6c596ec4c497 100644
--- a/proto/common.proto
+++ b/proto/common.proto
@@ -35,8 +35,11 @@ enum WorkerType {
   WORKER_TYPE_META = 5;
 }
 
-message ParallelUnit {
-  uint32 id = 1;
+// renamed from `ParallelUnit`
+message ActorLocation {
+  // previous `id` field for parallel unit id is deprecated.
+  reserved "id";
+  reserved 1;
   uint32 worker_node_id = 2;
 }
 
@@ -61,8 +64,10 @@ message WorkerNode {
   WorkerType type = 2;
   HostAddress host = 3;
   State state = 4;
-  // TODO #8940 `parallel_units` should be moved into `Property`
-  repeated ParallelUnit parallel_units = 5;
+
+  reserved 5;
+  reserved "parallel_units";
+
   Property property = 6;
 
   // Ranges from 0 to 1023, used to generate the machine ID field in the global unique ID.
@@ -75,6 +80,12 @@ message WorkerNode {
   // It's populated by meta node, when the worker node is added by meta node.
   // It's not persistent in meta store.
   optional uint64 started_at = 9;
+
+  uint32 parallelism = 10;
+
+  // Meta may assign labels to worker nodes to partition workload by label.
+  // This is used for serverless backfilling of materialized views.
+  string node_label = 11;
 }
 
 message Buffer {
@@ -86,12 +97,6 @@ message Buffer {
   bytes body = 2;
 }
 
-// Vnode mapping for stream fragments. Stores mapping from virtual node to parallel unit id.
-message ParallelUnitMapping {
-  repeated uint32 original_indices = 1;
-  repeated uint32 data = 2;
-}
-
 // Vnode mapping for stream fragments. Stores mapping from virtual node to (worker id, slot index).
 message WorkerSlotMapping {
   repeated uint32 original_indices = 1;
@@ -103,6 +108,7 @@ message BatchQueryEpoch {
     uint64 committed = 1;
     uint64 current = 2;
     uint64 backup = 3;
+    uint64 time_travel = 4;
   }
 }
 
diff --git a/proto/connector_service.proto b/proto/connector_service.proto
index da2c2b88087ea..964d227452548 100644
--- a/proto/connector_service.proto
+++ b/proto/connector_service.proto
@@ -147,13 +147,22 @@ message SinkCoordinatorStreamResponse {
 /* Source Service */
 
 message CdcMessage {
+  enum CdcMessageType {
+    UNSPECIFIED = 0;
+    HEARTBEAT = 1;
+    DATA = 2;
+    TRANSACTION_META = 3;
+    SCHEMA_CHANGE = 4;
+  }
+
   // The value of the Debezium message
   string payload = 1;
   string partition = 2;
   string offset = 3;
   string full_table_name = 4;
   int64 source_ts_ms = 5;
-  bool is_transaction_meta = 6;
+  CdcMessageType msg_type = 6;
+
   // The key of the Debezium message, which only used by `mongodb-cdc` connector.
   string key = 7;
 }
@@ -164,6 +173,7 @@ enum SourceType {
   POSTGRES = 2;
   CITUS = 3;
   MONGODB = 4;
+  SQL_SERVER = 5;
 }
 
 message GetEventStreamRequest {
diff --git a/proto/ddl_service.proto b/proto/ddl_service.proto
index 46c2a5c22ff6d..f78c08e2a9b52 100644
--- a/proto/ddl_service.proto
+++ b/proto/ddl_service.proto
@@ -5,6 +5,7 @@ package ddl_service;
 import "catalog.proto";
 import "common.proto";
 import "meta.proto";
+import "plan_common.proto";
 import "stream_plan.proto";
 
 option java_package = "com.risingwave.proto";
@@ -120,6 +121,16 @@ message DropSubscriptionResponse {
 message CreateMaterializedViewRequest {
   catalog.Table materialized_view = 1;
   stream_plan.StreamFragmentGraph fragment_graph = 2;
+
+  // If SERVERLESS, the materialized view should be created using serverless backfill
+  // For that the controller will create a new compute node, which does backfilling and then is deleted.
+  // May alleviate pressure on the cluster during backfill process.
+  enum BackfillType {
+    UNSPECIFIED = 0;
+    REGULAR = 1;
+    SERVERLESS = 2;
+  }
+  BackfillType backfill = 3;
 }
 
 message CreateMaterializedViewResponse {
@@ -324,6 +335,7 @@ message ReplaceTablePlan {
   catalog.ColIndexMapping table_col_index_mapping = 3;
   // Source catalog of table's associated source
   catalog.Source source = 4;
+  TableJobType job_type = 5;
 }
 
 message ReplaceTablePlanRequest {
@@ -433,6 +445,23 @@ message CommentOnResponse {
   uint64 version = 2;
 }
 
+message TableSchemaChange {
+  enum TableChangeType {
+    UNSPECIFIED = 0;
+    ALTER = 1;
+    CREATE = 2;
+    DROP = 3;
+  }
+
+  TableChangeType change_type = 1;
+  string cdc_table_name = 2;
+  repeated plan_common.ColumnCatalog columns = 3;
+}
+
+message SchemaChangeEnvelope {
+  repeated TableSchemaChange table_changes = 1;
+}
+
 service DdlService {
   rpc CreateDatabase(CreateDatabaseRequest) returns (CreateDatabaseResponse);
   rpc DropDatabase(DropDatabaseRequest) returns (DropDatabaseResponse);
diff --git a/proto/expr.proto b/proto/expr.proto
index 3bb54fae6f0bb..9887505bf61dc 100644
--- a/proto/expr.proto
+++ b/proto/expr.proto
@@ -154,6 +154,7 @@ message ExprNode {
     ACOSH = 265;
     ATANH = 266;
     SINH = 267;
+    ACOSD = 268;
     // skips 268,269,270 so that acosd, atand, atan2d are close to others
     TRUNC = 271;
     LN = 272;
@@ -307,6 +308,8 @@ message ExprNode {
     HAS_TABLE_PRIVILEGE = 2407;
     HAS_ANY_COLUMN_PRIVILEGE = 2408;
     HAS_SCHEMA_PRIVILEGE = 2409;
+    PG_IS_IN_RECOVERY = 2411;
+    RW_RECOVERY_STATUS = 2412;
 
     // EXTERNAL
     ICEBERG_TRANSFORM = 2201;
@@ -343,6 +346,8 @@ message TableFunction {
     JSONB_PATH_QUERY = 15;
     JSONB_POPULATE_RECORDSET = 16;
     JSONB_TO_RECORDSET = 17;
+    // file scan
+    FILE_SCAN = 19;
     // User defined table function
     USER_DEFINED = 100;
   }
@@ -432,8 +437,12 @@ message AggCall {
     LAST_VALUE = 25;
     GROUPING = 26;
     INTERNAL_LAST_SEEN_VALUE = 27;
+    APPROX_PERCENTILE = 28;
+
     // user defined aggregate function
     USER_DEFINED = 100;
+    // wraps a scalar function that takes a list as input as an aggregate function.
+    WRAP_SCALAR = 101;
   }
   Type type = 1;
   repeated InputRef args = 2;
@@ -444,6 +453,8 @@ message AggCall {
   repeated Constant direct_args = 7;
   // optional. only used when the type is USER_DEFINED.
   UserDefinedFunctionMetadata udf = 8;
+  // optional. only used when the type is WRAP_SCALAR.
+  ExprNode scalar = 9;
 }
 
 message WindowFrame {
@@ -575,6 +586,7 @@ message UserDefinedFunction {
 message UserDefinedFunctionMetadata {
   repeated string arg_names = 8;
   repeated data.DataType arg_types = 3;
+  data.DataType return_type = 13;
   string language = 4;
   optional string link = 5;
   optional string identifier = 6;
diff --git a/proto/hummock.proto b/proto/hummock.proto
index 149944831a4f9..5d66a2b7bb79b 100644
--- a/proto/hummock.proto
+++ b/proto/hummock.proto
@@ -104,7 +104,7 @@ message GroupDelta {
     IntraLevelDelta intra_level = 1;
     GroupConstruct group_construct = 2;
     GroupDestroy group_destroy = 3;
-    GroupMetaChange group_meta_change = 4;
+    GroupMetaChange group_meta_change = 4 [deprecated = true];
     GroupTableChange group_table_change = 5 [deprecated = true];
   }
 }
@@ -384,6 +384,7 @@ message CompactTask {
     TTL = 5;
     TOMBSTONE = 6;
     EMERGENCY = 7;
+    VNODE_WATERMARK = 8;
   }
 
   // Identifies whether the task is space_reclaim, if the compact_task_type increases, it will be refactored to enum
@@ -571,6 +572,7 @@ message VacuumTask {
 // Scan object store to get candidate orphan SSTs.
 message FullScanTask {
   uint64 sst_retention_time_sec = 1;
+  optional string prefix = 2;
 }
 
 // Cancel compact task
@@ -614,6 +616,7 @@ message ReportFullScanTaskResponse {
 
 message TriggerFullGCRequest {
   uint64 sst_retention_time_sec = 1;
+  optional string prefix = 2;
 }
 
 message TriggerFullGCResponse {
@@ -821,6 +824,14 @@ message CancelCompactTaskResponse {
   bool ret = 1;
 }
 
+message GetVersionByEpochRequest {
+  uint64 epoch = 1;
+}
+
+message GetVersionByEpochResponse {
+  HummockVersion version = 1;
+}
+
 service HummockManagerService {
   rpc UnpinVersionBefore(UnpinVersionBeforeRequest) returns (UnpinVersionBeforeResponse);
   rpc GetCurrentVersion(GetCurrentVersionRequest) returns (GetCurrentVersionResponse);
@@ -861,6 +872,7 @@ service HummockManagerService {
   rpc ListCompactTaskProgress(ListCompactTaskProgressRequest) returns (ListCompactTaskProgressResponse);
   rpc CancelCompactTask(CancelCompactTaskRequest) returns (CancelCompactTaskResponse);
   rpc ListChangeLogEpochs(ListChangeLogEpochsRequest) returns (ListChangeLogEpochsResponse);
+  rpc GetVersionByEpoch(GetVersionByEpochRequest) returns (GetVersionByEpochResponse);
 }
 
 message CompactionConfig {
@@ -894,7 +906,7 @@ message CompactionConfig {
   bool enable_emergency_picker = 20;
 
   // The limitation of the level count of l0 compaction
-  uint32 max_l0_compact_level_count = 21;
+  optional uint32 max_l0_compact_level_count = 21;
 }
 
 message TableStats {
diff --git a/proto/meta.proto b/proto/meta.proto
index e4068a0b8cd58..0371b5540a6da 100644
--- a/proto/meta.proto
+++ b/proto/meta.proto
@@ -65,8 +65,8 @@ message TableFragments {
       // Running normally
       RUNNING = 2;
     }
-    // Current on which parallel unit
-    common.ParallelUnit parallel_unit = 1;
+    // Current on which worker
+    common.ActorLocation location = 1;
     // Current state
     ActorState state = 2;
   }
@@ -81,9 +81,13 @@ message TableFragments {
     uint32 fragment_type_mask = 2;
     FragmentDistributionType distribution_type = 3;
     repeated stream_plan.StreamActor actors = 4;
+
+    // NOTE: vnode_mapping is deprecated, we will generate the vnode_mapping by actors' bitmaps
     // Vnode mapping (which should be set in upstream dispatcher) of the fragment.
     // This field is always set to `Some`. For singleton, the parallel unit for all vnodes will be the same.
-    common.ParallelUnitMapping vnode_mapping = 5;
+    reserved 5;
+    reserved "vnode_mapping";
+
     repeated uint32 state_table_ids = 6;
     // Note that this can be derived backwards from the upstream actors of the Actor held by the Fragment,
     // but in some scenarios (e.g. Scaling) it will lead to a lot of duplicate code,
@@ -99,16 +103,13 @@ message TableFragments {
 
   stream_plan.StreamContext ctx = 6;
   TableParallelism parallelism = 7;
-}
 
-/// Parallel unit mapping with fragment id, used for notification.
-message FragmentParallelUnitMapping {
-  uint32 fragment_id = 1;
-  common.ParallelUnitMapping mapping = 2;
-}
+  // Actors of a materialize view, sink, or table can only be scheduled on nodes with matching node_label.
+  string node_label = 8;
 
-message FragmentParallelUnitMappings {
-  repeated FragmentParallelUnitMapping mappings = 1;
+  // If this is a materialized view: True if backfill is done, else false.
+  // If this is a regular table: Always true.
+  bool backfill_done = 9;
 }
 
 /// Worker slot mapping with fragment id, used for notification.
@@ -128,8 +129,13 @@ message ActorLocation {
 }
 
 message MigrationPlan {
+  // NOTE: parallel_unit_migration_plan is deprecated, using worker_slot_migration_plan instead
   // map<parallel_unit_id, parallel_unit>, the plan indicates that the actors will be migrated from old parallel unit to the new one.
-  map<uint32, common.ParallelUnit> parallel_unit_migration_plan = 1;
+  reserved 1;
+  reserved "parallel_unit_migration_plan";
+
+  // map<src_worker_slot_id, dst_worker_slot_id>, the plan indicates that the actors will be migrated from old worker_slot to the new one.
+  map<uint64, uint64> worker_slot_migration_plan = 2;
 }
 
 message FlushRequest {
@@ -244,8 +250,10 @@ message ListActorStatesResponse {
   message ActorState {
     uint32 actor_id = 1;
     uint32 fragment_id = 2;
-    uint32 parallel_unit_id = 3;
+    reserved 3;
+    reserved "parallel_unit_id";
     TableFragments.ActorStatus.ActorState state = 4;
+    uint32 worker_id = 5;
   }
   repeated ActorState states = 1;
 }
@@ -316,6 +324,7 @@ message AddWorkerNodeResponse {
   reserved "system_params";
   common.Status status = 1;
   optional uint32 node_id = 2;
+  string cluster_id = 4;
 }
 
 message ActivateWorkerNodeRequest {
@@ -362,12 +371,26 @@ message ListAllNodesResponse {
   repeated common.WorkerNode nodes = 2;
 }
 
+message GetClusterRecoveryStatusRequest {}
+
+enum RecoveryStatus {
+  STATUS_UNSPECIFIED = 0;
+  STATUS_STARTING = 1;
+  STATUS_RECOVERING = 2;
+  STATUS_RUNNING = 3;
+}
+
+message GetClusterRecoveryStatusResponse {
+  RecoveryStatus status = 1;
+}
+
 service ClusterService {
   rpc AddWorkerNode(AddWorkerNodeRequest) returns (AddWorkerNodeResponse);
   rpc ActivateWorkerNode(ActivateWorkerNodeRequest) returns (ActivateWorkerNodeResponse);
   rpc DeleteWorkerNode(DeleteWorkerNodeRequest) returns (DeleteWorkerNodeResponse);
   rpc UpdateWorkerNodeSchedulability(UpdateWorkerNodeSchedulabilityRequest) returns (UpdateWorkerNodeSchedulabilityResponse);
   rpc ListAllNodes(ListAllNodesRequest) returns (ListAllNodesResponse);
+  rpc GetClusterRecoveryStatus(GetClusterRecoveryStatusRequest) returns (GetClusterRecoveryStatusResponse);
 }
 
 enum SubscribeType {
@@ -450,7 +473,10 @@ message SubscribeResponse {
   }
   common.Status status = 1;
   Operation operation = 2;
+
+  // Catalog version
   uint64 version = 3;
+
   oneof info {
     catalog.Database database = 4;
     catalog.Schema schema = 5;
@@ -492,16 +518,19 @@ message GetClusterInfoResponse {
   uint64 revision = 5;
 }
 
-message Reschedule {
-  repeated uint32 added_parallel_units = 1;
-  repeated uint32 removed_parallel_units = 2;
+// For each fragment that needs to be rescheduled, there will be a WorkerReschedule,
+// indicating on which workers the actors of this fragment need to be changed and by how many.
+message WorkerReschedule {
+  // worker_id -> actor_diff
+  map<uint32, int32> worker_actor_diff = 1;
 }
 
 message RescheduleRequest {
-  // reschedule plan for each fragment
-  map<uint32, Reschedule> reschedules = 1;
+  reserved "reschedules";
+  reserved 1;
   uint64 revision = 2;
   bool resolve_no_shuffle_upstream = 3;
+  map<uint32, WorkerReschedule> worker_reschedules = 4;
 }
 
 message RescheduleResponse {
@@ -529,40 +558,38 @@ message TableParallelism {
   }
 }
 
-message GetReschedulePlanRequest {
-  uint64 revision = 1;
+// Changes a streaming job in place by overwriting its node_label.
+// This may cause the re-scheduling of the streaming job actors.
+message UpdateStreamingJobNodeLabelsRequest {
+  // Id of the materialized view, table, or sink which we want to update
+  uint32 id = 1;
 
-  message WorkerChanges {
-    repeated uint32 include_worker_ids = 1;
-    repeated uint32 exclude_worker_ids = 2;
-    optional uint32 target_parallelism = 3;
-    optional uint32 target_parallelism_per_worker = 4;
-  }
+  // replace the node_label of the streaming job with a given id with below value
+  string node_label = 2;
+}
 
-  message StableResizePolicy {
-    map<uint32, WorkerChanges> fragment_worker_changes = 1;
-  }
+// We do not need to add an explicit status field here, we can just use the RPC status
+message UpdateStreamingJobNodeLabelsResponse {}
+
+message GetServerlessStreamingJobsStatusRequest {}
 
-  oneof policy {
-    // The StableResizePolicy will generate a stable ReschedulePlan, without altering the distribution on WorkerId that's not involved.
-    // Note that this "Stable" doesn't refer to the "determinacy" of the algorithm.
-    // Multiple repeated calls may yield different ReschedulePlan results.
-    StableResizePolicy stable_resize_policy = 2;
+// Descriptions of MVs and sinks
+message GetServerlessStreamingJobsStatusResponse {
+  message Status {
+    uint32 table_id = 1;
+    string node_label = 2;
+    bool backfill_done = 3;
   }
-}
 
-message GetReschedulePlanResponse {
-  uint64 revision = 1;
-  // reschedule plan for each fragment
-  map<uint32, Reschedule> reschedules = 2;
-  // todo, refactor needed
-  bool success = 3;
+  repeated Status streaming_job_statuses = 1;
 }
 
+// This is used by `risectl`
 service ScaleService {
   rpc GetClusterInfo(GetClusterInfoRequest) returns (GetClusterInfoResponse);
   rpc Reschedule(RescheduleRequest) returns (RescheduleResponse);
-  rpc GetReschedulePlan(GetReschedulePlanRequest) returns (GetReschedulePlanResponse);
+  rpc UpdateStreamingJobNodeLabels(UpdateStreamingJobNodeLabelsRequest) returns (UpdateStreamingJobNodeLabelsResponse);
+  rpc GetServerlessStreamingJobsStatus(GetServerlessStreamingJobsStatusRequest) returns (GetServerlessStreamingJobsStatusResponse);
 }
 
 message MembersRequest {}
@@ -604,6 +631,7 @@ message SystemParams {
   optional bool enable_tracing = 15;
   optional bool use_new_object_prefix_strategy = 16;
   optional string license_key = 17;
+  optional uint64 time_travel_retention_ms = 18;
 }
 
 message GetSystemParamsRequest {}
diff --git a/proto/plan_common.proto b/proto/plan_common.proto
index 7b763c24a44a8..bc2e60503f103 100644
--- a/proto/plan_common.proto
+++ b/proto/plan_common.proto
@@ -98,6 +98,21 @@ message StorageTableDesc {
   optional uint32 retention_seconds = 11;
 }
 
+message AsOf {
+  message ProcessTime {}
+  message Timestamp {
+    int64 timestamp = 1;
+  }
+  message Version {
+    int64 version = 1;
+  }
+  oneof as_of_type {
+    ProcessTime process_time = 1;
+    Timestamp timestamp = 2;
+    Version version = 3;
+  }
+}
+
 // Represents a table in external database for CDC scenario
 message ExternalTableDesc {
   uint32 table_id = 1;
@@ -150,6 +165,7 @@ enum EncodeType {
   ENCODE_TYPE_TEMPLATE = 7;
   ENCODE_TYPE_NONE = 8;
   ENCODE_TYPE_TEXT = 9;
+  ENCODE_TYPE_PARQUET = 10;
 }
 
 enum RowFormatType {
diff --git a/proto/stream_plan.proto b/proto/stream_plan.proto
index 3469851c566b7..7aa48d1184d03 100644
--- a/proto/stream_plan.proto
+++ b/proto/stream_plan.proto
@@ -28,6 +28,7 @@ message AddMutation {
   // We may embed a pause mutation here.
   // TODO: we may allow multiple mutations in a single barrier.
   bool pause = 4;
+  repeated SubscriptionUpstreamInfo subscriptions_to_add = 5;
 }
 
 message StopMutation {
@@ -95,14 +96,13 @@ message CombinedMutation {
   repeated BarrierMutation mutations = 1;
 }
 
-message CreateSubscriptionMutation {
-  uint32 subscription_id = 1;
+message SubscriptionUpstreamInfo {
+  uint32 subscriber_id = 1;
   uint32 upstream_mv_table_id = 2;
 }
 
-message DropSubscriptionMutation {
-  uint32 subscription_id = 1;
-  uint32 upstream_mv_table_id = 2;
+message DropSubscriptionsMutation {
+  repeated SubscriptionUpstreamInfo info = 1;
 }
 
 message BarrierMutation {
@@ -122,10 +122,8 @@ message BarrierMutation {
     ResumeMutation resume = 8;
     // Throttle specific source exec or chain exec.
     ThrottleMutation throttle = 10;
-    // Create subscription on mv
-    CreateSubscriptionMutation create_subscription = 11;
     // Drop subscription on mv
-    DropSubscriptionMutation drop_subscription = 12;
+    DropSubscriptionsMutation drop_subscriptions = 12;
     // Combined mutation.
     CombinedMutation combined = 100;
   }
@@ -480,7 +478,7 @@ message DynamicFilterNode {
   // When the condition changes, we will tell downstream to insert the LHS records which now match the condition.
   // If this is false, we need to store RHS records which match the condition in the internal table.
   // When the condition changes, we will tell downstream to delete the LHS records which now no longer match the condition.
-  bool condition_always_relax = 5;
+  bool condition_always_relax = 5 [deprecated = true];
 }
 
 // Delta join with two indexes. This is a pseudo plan node generated on frontend. On meta
@@ -790,6 +788,23 @@ message OverWindowNode {
   OverWindowCachePolicy cache_policy = 5;
 }
 
+message LocalApproxPercentileNode {
+  double base = 1;
+  uint32 percentile_index = 2;
+}
+
+message GlobalApproxPercentileNode {
+  double base = 1;
+  double quantile = 2;
+  catalog.Table bucket_state_table = 3;
+  catalog.Table count_state_table = 4;
+}
+
+message RowMergeNode {
+  catalog.ColIndexMapping lhs_mapping = 1;
+  catalog.ColIndexMapping rhs_mapping = 2;
+}
+
 message StreamNode {
   oneof node_body {
     SourceNode source = 100;
@@ -835,6 +850,9 @@ message StreamNode {
     CdcFilterNode cdc_filter = 140;
     SourceBackfillNode source_backfill = 142;
     ChangeLogNode changelog = 143;
+    LocalApproxPercentileNode local_approx_percentile = 144;
+    GlobalApproxPercentileNode global_approx_percentile = 145;
+    RowMergeNode row_merge = 146;
   }
   // The id for the operator. This is local per mview.
   // TODO: should better be a uint32.
diff --git a/proto/stream_service.proto b/proto/stream_service.proto
index 85b12d8ed5fa1..08f0ff1e7684f 100644
--- a/proto/stream_service.proto
+++ b/proto/stream_service.proto
@@ -58,6 +58,7 @@ message InjectBarrierRequest {
   repeated uint32 actor_ids_to_send = 3;
   repeated uint32 actor_ids_to_collect = 4;
   repeated uint32 table_ids_to_sync = 5;
+  uint32 partial_graph_id = 6;
 }
 
 message BarrierCompleteResponse {
@@ -71,7 +72,8 @@ message BarrierCompleteResponse {
   common.Status status = 2;
   repeated CreateMviewProgress create_mview_progress = 3;
   message GroupedSstableInfo {
-    uint64 compaction_group_id = 1;
+    reserved 1;
+    reserved "compaction_group_id";
     hummock.SstableInfo sst = 2;
     map<uint32, hummock.TableStats> table_stats_map = 3;
   }
@@ -79,6 +81,9 @@ message BarrierCompleteResponse {
   uint32 worker_id = 5;
   map<uint32, hummock.TableWatermarks> table_watermarks = 6;
   repeated hummock.SstableInfo old_value_sstables = 7;
+  uint32 partial_graph_id = 8;
+  // prev_epoch of barrier
+  uint64 epoch = 9;
 }
 
 // Before starting streaming, the leader node broadcast the actor-host table to needed workers.
@@ -99,18 +104,25 @@ message StreamingControlStreamRequest {
     uint64 prev_epoch = 2;
   }
 
+  message RemovePartialGraphRequest {
+    repeated uint32 partial_graph_ids = 1;
+  }
+
   oneof request {
     InitRequest init = 1;
     InjectBarrierRequest inject_barrier = 2;
+    RemovePartialGraphRequest remove_partial_graph = 3;
   }
 }
 
 message StreamingControlStreamResponse {
   message InitResponse {}
+  message ShutdownResponse {}
 
   oneof response {
     InitResponse init = 1;
     BarrierCompleteResponse complete_barrier = 2;
+    ShutdownResponse shutdown = 3;
   }
 }
 
diff --git a/proto/telemetry.proto b/proto/telemetry.proto
index a3c4ab7ef9dfa..e9d5b75147e5e 100644
--- a/proto/telemetry.proto
+++ b/proto/telemetry.proto
@@ -125,3 +125,44 @@ message FrontendReport {
 message CompactorReport {
   ReportBase base = 1;
 }
+
+enum TelemetryEventStage {
+  TELEMETRY_EVENT_STAGE_UNSPECIFIED = 0;
+  TELEMETRY_EVENT_STAGE_CREATE_STREAM_JOB = 1;
+  TELEMETRY_EVENT_STAGE_UPDATE_STREAM_JOB = 2;
+  TELEMETRY_EVENT_STAGE_DROP_STREAM_JOB = 3;
+  TELEMETRY_EVENT_STAGE_QUERY = 4;
+  TELEMETRY_EVENT_STAGE_RECOVERY = 5;
+}
+
+enum TelemetryDatabaseObject {
+  TELEMETRY_DATABASE_OBJECT_UNSPECIFIED = 0;
+  TELEMETRY_DATABASE_OBJECT_SOURCE = 1;
+  TELEMETRY_DATABASE_OBJECT_MV = 2;
+  TELEMETRY_DATABASE_OBJECT_TABLE = 3;
+  TELEMETRY_DATABASE_OBJECT_SINK = 4;
+  TELEMETRY_DATABASE_OBJECT_INDEX = 5;
+}
+
+message EventMessage {
+  // tracking_id is persistent in meta data
+  string tracking_id = 1;
+  // event_time is when the event is created
+  uint64 event_time_sec = 3;
+  // event_stage describes in which process the event happens
+  TelemetryEventStage event_stage = 4;
+  // feature_name is the name of the feature triggered the event
+  string event_name = 5;
+  // connector_name is the name of the connector involves
+  optional string connector_name = 6;
+  // connector_direction is the direction of data flow, can be source or sink
+  optional TelemetryDatabaseObject object = 10;
+  // catalog_id is the id of the catalog involves (table_id/source_id/...)
+  int64 catalog_id = 7;
+  // attributes is the additional information of the event: json format ser to string
+  optional string attributes = 8;
+  // node is the node that creates the event
+  string node = 9;
+  // mark the event is a test message
+  bool is_test = 11;
+}
diff --git a/risedev.yml b/risedev.yml
index cdc115ca30010..db8f6fe5600e2 100644
--- a/risedev.yml
+++ b/risedev.yml
@@ -296,27 +296,14 @@ profile:
         port: 15690
         dashboard-port: 15691
         exporter-port: 11250
+        meta-backend: etcd
       - use: meta-node
         port: 25690
         dashboard-port: 25691
         exporter-port: 21250
+        meta-backend: etcd
       - use: compactor
 
-  3meta:
-    steps:
-      - use: meta-node
-        port: 5690
-        dashboard-port: 5691
-        exporter-port: 1250
-      - use: meta-node
-        port: 15690
-        dashboard-port: 15691
-        exporter-port: 11250
-      - use: meta-node
-        port: 25690
-        dashboard-port: 25691
-        exporter-port: 21250
-
   3etcd-3meta-1cn-1fe:
     steps:
       - use: minio
@@ -344,10 +331,12 @@ profile:
         port: 15690
         dashboard-port: 15691
         exporter-port: 11250
+        meta-backend: etcd
       - use: meta-node
         port: 25690
         dashboard-port: 25691
         exporter-port: 21250
+        meta-backend: etcd
       - use: compactor
       - use: compute-node
       - use: frontend
@@ -365,6 +354,20 @@ profile:
       - use: compute-node
       - use: frontend
 
+  ci-time-travel:
+    config-path: src/config/ci-time-travel.toml
+    steps:
+      - use: minio
+      - use: sqlite
+      - use: meta-node
+        port: 5690
+        dashboard-port: 5691
+        exporter-port: 1250
+        meta-backend: sqlite
+      - use: compactor
+      - use: compute-node
+      - use: frontend
+
   meta-1cn-1fe-sqlite-with-recovery:
     config-path: src/config/ci-recovery.toml
     steps:
@@ -1618,3 +1621,33 @@ template:
     # If `user-managed` is true, user is responsible for starting the service
     # to serve at the above address and port in any way they see fit.
     user-managed: false
+
+  # Sql Server service backed by docker.
+  sqlserver:
+    # Note: Sql Server is now only for connector purpose.
+    # Id to be picked-up by services
+    id: sqlserver-${port}
+
+    # address of mssql
+    address: "127.0.0.1"
+
+    # listen port of mssql
+    port: 1433
+
+    # Note:
+    # - This will be used to initialize the Sql Server instance if it's fresh.
+    # - In user-managed mode, these configs are not validated by risedev.
+    #   They are passed as-is to risedev-env default user for Sql Server operations.
+    user: SA
+    password: "YourPassword123"
+    database: "master"
+
+    # The docker image. Can be overridden to use a different version.
+    image: "mcr.microsoft.com/mssql/server:2022-latest"
+
+    # If set to true, data will be persisted at data/{id}.
+    persist-data: true
+
+    # If `user-managed` is true, user is responsible for starting the service
+    # to serve at the above address and port in any way they see fit.
+    user-managed: false
diff --git a/src/batch/Cargo.toml b/src/batch/Cargo.toml
index cf94b2ec838aa..ec7091ea882c4 100644
--- a/src/batch/Cargo.toml
+++ b/src/batch/Cargo.toml
@@ -32,8 +32,9 @@ hytra = "0.1.2"
 iceberg = { workspace = true }
 itertools = { workspace = true }
 memcomparable = "0.2"
-opendal = "0.47"
+opendal = { workspace = true }
 parking_lot = { workspace = true }
+parquet = { workspace = true }
 paste = "1"
 prometheus = { version = "0.13", features = ["process"] }
 prost = "0.12"
diff --git a/src/batch/benches/hash_agg.rs b/src/batch/benches/hash_agg.rs
index d86812623d422..e393f1ff1bf8c 100644
--- a/src/batch/benches/hash_agg.rs
+++ b/src/batch/benches/hash_agg.rs
@@ -25,7 +25,7 @@ use risingwave_common::catalog::{Field, Schema};
 use risingwave_common::memory::MemoryContext;
 use risingwave_common::types::DataType;
 use risingwave_common::{enable_jemalloc, hash};
-use risingwave_expr::aggregate::{AggCall, AggKind};
+use risingwave_expr::aggregate::{AggCall, AggKind, PbAggKind};
 use risingwave_pb::expr::{PbAggCall, PbInputRef};
 use tokio::runtime::Runtime;
 use utils::{create_input, execute_executor};
@@ -53,6 +53,7 @@ fn create_agg_call(
         filter: None,
         direct_args: vec![],
         udf: None,
+        scalar: None,
     }
 }
 
@@ -119,15 +120,15 @@ fn bench_hash_agg(c: &mut Criterion) {
 
     let bench_variants = [
         // (group by, agg, args, return type)
-        (vec![0], AggKind::Sum, vec![1], DataType::Int64),
-        (vec![0], AggKind::Count, vec![], DataType::Int64),
-        (vec![0], AggKind::Count, vec![2], DataType::Int64),
-        (vec![0], AggKind::Min, vec![1], DataType::Int64),
-        (vec![0], AggKind::StringAgg, vec![2], DataType::Varchar),
-        (vec![0, 2], AggKind::Sum, vec![1], DataType::Int64),
-        (vec![0, 2], AggKind::Count, vec![], DataType::Int64),
-        (vec![0, 2], AggKind::Count, vec![2], DataType::Int64),
-        (vec![0, 2], AggKind::Min, vec![1], DataType::Int64),
+        (vec![0], PbAggKind::Sum, vec![1], DataType::Int64),
+        (vec![0], PbAggKind::Count, vec![], DataType::Int64),
+        (vec![0], PbAggKind::Count, vec![2], DataType::Int64),
+        (vec![0], PbAggKind::Min, vec![1], DataType::Int64),
+        (vec![0], PbAggKind::StringAgg, vec![2], DataType::Varchar),
+        (vec![0, 2], PbAggKind::Sum, vec![1], DataType::Int64),
+        (vec![0, 2], PbAggKind::Count, vec![], DataType::Int64),
+        (vec![0, 2], PbAggKind::Count, vec![2], DataType::Int64),
+        (vec![0, 2], PbAggKind::Min, vec![1], DataType::Int64),
     ];
 
     for (group_key_columns, agg_kind, arg_columns, return_type) in bench_variants {
@@ -141,7 +142,7 @@ fn bench_hash_agg(c: &mut Criterion) {
                         || {
                             create_hash_agg_executor(
                                 group_key_columns.clone(),
-                                agg_kind,
+                                agg_kind.into(),
                                 arg_columns.clone(),
                                 return_type.clone(),
                                 chunk_size,
diff --git a/src/batch/src/error.rs b/src/batch/src/error.rs
index 2a555f18c356a..780b90cf178b0 100644
--- a/src/batch/src/error.rs
+++ b/src/batch/src/error.rs
@@ -17,6 +17,7 @@
 use std::sync::Arc;
 
 pub use anyhow::anyhow;
+use parquet::errors::ParquetError;
 use risingwave_common::array::ArrayError;
 use risingwave_common::error::BoxedError;
 use risingwave_common::util::value_encoding::error::ValueEncodingError;
@@ -119,6 +120,13 @@ pub enum BatchError {
         iceberg::Error,
     ),
 
+    #[error(transparent)]
+    Parquet(
+        #[from]
+        #[backtrace]
+        ParquetError,
+    ),
+
     // Make the ref-counted type to be a variant for easier code structuring.
     // TODO(error-handling): replace with `thiserror_ext::Arc`
     #[error(transparent)]
@@ -146,6 +154,13 @@ pub enum BatchError {
         #[backtrace]
         opendal::Error,
     ),
+
+    #[error("Failed to execute time travel query")]
+    TimeTravel(
+        #[source]
+        #[backtrace]
+        anyhow::Error,
+    ),
 }
 
 // Serialize/deserialize error.
diff --git a/src/batch/src/executor/aggregation/distinct.rs b/src/batch/src/executor/aggregation/distinct.rs
index 728ea2631b498..97780271b83fd 100644
--- a/src/batch/src/executor/aggregation/distinct.rs
+++ b/src/batch/src/executor/aggregation/distinct.rs
@@ -16,7 +16,7 @@ use std::collections::HashSet;
 use std::ops::Range;
 
 use risingwave_common::array::StreamChunk;
-use risingwave_common::buffer::BitmapBuilder;
+use risingwave_common::bitmap::BitmapBuilder;
 use risingwave_common::row::{OwnedRow, Row};
 use risingwave_common::types::{DataType, Datum};
 use risingwave_common_estimate_size::EstimateSize;
diff --git a/src/batch/src/executor/hash_agg.rs b/src/batch/src/executor/hash_agg.rs
index 6929e31898b65..d69d4fbc8b174 100644
--- a/src/batch/src/executor/hash_agg.rs
+++ b/src/batch/src/executor/hash_agg.rs
@@ -22,7 +22,7 @@ use hashbrown::hash_map::Entry;
 use itertools::Itertools;
 use prost::Message;
 use risingwave_common::array::{DataChunk, StreamChunk};
-use risingwave_common::buffer::Bitmap;
+use risingwave_common::bitmap::Bitmap;
 use risingwave_common::catalog::{Field, Schema};
 use risingwave_common::hash::{HashKey, HashKeyDispatcher, PrecomputedBuildHasher};
 use risingwave_common::memory::MemoryContext;
@@ -805,6 +805,7 @@ mod tests {
                 filter: None,
                 direct_args: vec![],
                 udf: None,
+                scalar: None,
             };
 
             let agg_prost = HashAggNode {
@@ -883,6 +884,7 @@ mod tests {
             filter: None,
             direct_args: vec![],
             udf: None,
+            scalar: None,
         };
 
         let agg_prost = HashAggNode {
@@ -1000,6 +1002,7 @@ mod tests {
             filter: None,
             direct_args: vec![],
             udf: None,
+            scalar: None,
         };
 
         let agg_prost = HashAggNode {
@@ -1092,6 +1095,7 @@ mod tests {
             filter: None,
             direct_args: vec![],
             udf: None,
+            scalar: None,
         };
 
         let agg_prost = HashAggNode {
diff --git a/src/batch/src/executor/iceberg_scan.rs b/src/batch/src/executor/iceberg_scan.rs
index 1ea9599dad69f..ee4e463422c14 100644
--- a/src/batch/src/executor/iceberg_scan.rs
+++ b/src/batch/src/executor/iceberg_scan.rs
@@ -12,12 +12,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::future;
-use std::hash::{DefaultHasher, Hash, Hasher};
+use std::mem;
 
-use anyhow::anyhow;
 use futures_async_stream::try_stream;
 use futures_util::stream::StreamExt;
+use iceberg::scan::FileScanTask;
+use iceberg::spec::TableMetadata;
 use risingwave_common::array::arrow::IcebergArrowConvert;
 use risingwave_common::catalog::Schema;
 use risingwave_connector::sink::iceberg::IcebergConfig;
@@ -25,57 +25,13 @@ use risingwave_connector::sink::iceberg::IcebergConfig;
 use crate::error::BatchError;
 use crate::executor::{DataChunk, Executor};
 
-/// Create a iceberg scan executor.
-///
-/// # Examples
-///
-/// ```
-/// use futures_async_stream::for_await;
-/// use risingwave_batch::executor::{Executor, FileSelector, IcebergScanExecutor};
-/// use risingwave_common::catalog::{Field, Schema};
-/// use risingwave_common::types::DataType;
-/// use risingwave_connector::sink::iceberg::IcebergConfig;
-///
-/// #[tokio::test]
-/// async fn test_iceberg_scan() {
-///     let iceberg_scan_executor = IcebergScanExecutor::new(
-///         IcebergConfig {
-///             database_name: Some("demo_db".into()),
-///             table_name: "demo_table".into(),
-///             catalog_type: Some("storage".into()),
-///             path: "s3a://hummock001/".into(),
-///             endpoint: Some("http://127.0.0.1:9301".into()),
-///             access_key: "hummockadmin".into(),
-///             secret_key: "hummockadmin".into(),
-///             region: Some("us-east-1".into()),
-///             ..Default::default()
-///         },
-///         None,
-///         FileSelector::select_all(),
-///         1024,
-///         Schema::new(vec![
-///             Field::with_name(DataType::Int64, "seq_id"),
-///             Field::with_name(DataType::Int64, "user_id"),
-///             Field::with_name(DataType::Varchar, "user_name"),
-///         ]),
-///         "iceberg_scan".into(),
-///     );
-///
-///     let stream = Box::new(iceberg_scan_executor).execute();
-///     #[for_await]
-///     for chunk in stream {
-///         let chunk = chunk.unwrap();
-///         println!("{:?}", chunk);
-///     }
-/// }
-/// ```
-
 pub struct IcebergScanExecutor {
     iceberg_config: IcebergConfig,
+    #[allow(dead_code)]
     snapshot_id: Option<i64>,
-    file_selector: FileSelector,
+    table_meta: TableMetadata,
+    file_scan_tasks: Vec<FileScanTask>,
     batch_size: usize,
-
     schema: Schema,
     identity: String,
 }
@@ -98,7 +54,8 @@ impl IcebergScanExecutor {
     pub fn new(
         iceberg_config: IcebergConfig,
         snapshot_id: Option<i64>,
-        file_selector: FileSelector,
+        table_meta: TableMetadata,
+        file_scan_tasks: Vec<FileScanTask>,
         batch_size: usize,
         schema: Schema,
         identity: String,
@@ -106,7 +63,8 @@ impl IcebergScanExecutor {
         Self {
             iceberg_config,
             snapshot_id,
-            file_selector,
+            table_meta,
+            file_scan_tasks,
             batch_size,
             schema,
             identity,
@@ -114,40 +72,23 @@ impl IcebergScanExecutor {
     }
 
     #[try_stream(ok = DataChunk, error = BatchError)]
-    async fn do_execute(self: Box<Self>) {
-        let table = self.iceberg_config.load_table_v2().await?;
-
-        let snapshot_id = if let Some(snapshot_id) = self.snapshot_id {
-            snapshot_id
-        } else {
-            table
-                .metadata()
-                .current_snapshot()
-                .ok_or_else(|| {
-                    BatchError::Internal(anyhow!("No snapshot found for iceberg table"))
-                })?
-                .snapshot_id()
+    async fn do_execute(mut self: Box<Self>) {
+        let table = self
+            .iceberg_config
+            .load_table_v2_with_metadata(self.table_meta)
+            .await?;
+        let data_types = self.schema.data_types();
+
+        let file_scan_tasks = mem::take(&mut self.file_scan_tasks);
+
+        let file_scan_stream = {
+            #[try_stream]
+            async move {
+                for file_scan_task in file_scan_tasks {
+                    yield file_scan_task;
+                }
+            }
         };
-        let scan = table
-            .scan()
-            .snapshot_id(snapshot_id)
-            .with_batch_size(Some(self.batch_size))
-            .select(self.schema.names())
-            .build()
-            .map_err(|e| BatchError::Internal(anyhow!(e)))?;
-
-        let file_selector = self.file_selector.clone();
-        let file_scan_stream = scan
-            .plan_files()
-            .await
-            .map_err(BatchError::Iceberg)?
-            .filter(move |task| {
-                let res = task
-                    .as_ref()
-                    .map(|task| file_selector.select(task.data_file_path()))
-                    .unwrap_or(true);
-                future::ready(res)
-            });
 
         let reader = table
             .reader_builder()
@@ -162,39 +103,8 @@ impl IcebergScanExecutor {
         for record_batch in record_batch_stream {
             let record_batch = record_batch.map_err(BatchError::Iceberg)?;
             let chunk = IcebergArrowConvert.chunk_from_record_batch(&record_batch)?;
-            debug_assert_eq!(chunk.data_types(), self.schema.data_types());
+            debug_assert_eq!(chunk.data_types(), data_types);
             yield chunk;
         }
     }
 }
-
-#[derive(Clone)]
-pub enum FileSelector {
-    // File paths to be scanned by this executor are specified.
-    FileList(Vec<String>),
-    // Data files to be scanned by this executor could be calculated by Hash(file_path) % num_tasks == task_id.
-    // task_id, num_tasks
-    Hash(usize, usize),
-}
-
-impl FileSelector {
-    pub fn select_all() -> Self {
-        FileSelector::Hash(0, 1)
-    }
-
-    pub fn select(&self, path: &str) -> bool {
-        match self {
-            FileSelector::FileList(paths) => paths.contains(&path.to_string()),
-            FileSelector::Hash(task_id, num_tasks) => {
-                let hash = Self::hash_str_to_usize(path);
-                hash % num_tasks == *task_id
-            }
-        }
-    }
-
-    pub fn hash_str_to_usize(s: &str) -> usize {
-        let mut hasher = DefaultHasher::new();
-        s.hash(&mut hasher);
-        hasher.finish() as usize
-    }
-}
diff --git a/src/batch/src/executor/join/distributed_lookup_join.rs b/src/batch/src/executor/join/distributed_lookup_join.rs
index 1ff32d9631a8f..f5ad5ab5ed984 100644
--- a/src/batch/src/executor/join/distributed_lookup_join.rs
+++ b/src/batch/src/executor/join/distributed_lookup_join.rs
@@ -36,8 +36,8 @@ use risingwave_storage::{dispatch_state_store, StateStore};
 use crate::error::Result;
 use crate::executor::join::JoinType;
 use crate::executor::{
-    BoxedDataChunkStream, BoxedExecutor, BoxedExecutorBuilder, BufferChunkExecutor, Executor,
-    ExecutorBuilder, LookupExecutorBuilder, LookupJoinBase,
+    unix_timestamp_sec_to_epoch, AsOf, BoxedDataChunkStream, BoxedExecutor, BoxedExecutorBuilder,
+    BufferChunkExecutor, Executor, ExecutorBuilder, LookupExecutorBuilder, LookupJoinBase,
 };
 use crate::task::{BatchTaskContext, ShutdownToken};
 
@@ -93,6 +93,24 @@ impl BoxedExecutorBuilder for DistributedLookupJoinExecutorBuilder {
             NodeBody::DistributedLookupJoin
         )?;
 
+        // as_of takes precedence
+        let as_of = distributed_lookup_join_node
+            .as_of
+            .as_ref()
+            .map(AsOf::try_from)
+            .transpose()?;
+        let query_epoch = as_of
+            .map(|a| {
+                let epoch = unix_timestamp_sec_to_epoch(a.timestamp).0;
+                tracing::debug!(epoch, "time travel");
+                risingwave_pb::common::BatchQueryEpoch {
+                    epoch: Some(risingwave_pb::common::batch_query_epoch::Epoch::TimeTravel(
+                        epoch,
+                    )),
+                }
+            })
+            .unwrap_or_else(|| source.epoch());
+
         let join_type = JoinType::from_prost(distributed_lookup_join_node.get_join_type()?);
         let condition = match distributed_lookup_join_node.get_condition() {
             Ok(cond_prost) => Some(build_from_prost(cond_prost)?),
@@ -179,12 +197,11 @@ impl BoxedExecutorBuilder for DistributedLookupJoinExecutorBuilder {
         let vnodes = Some(TableDistribution::all_vnodes());
         dispatch_state_store!(source.context().state_store(), state_store, {
             let table = StorageTable::new_partial(state_store, column_ids, vnodes, table_desc);
-
             let inner_side_builder = InnerSideExecutorBuilder::new(
                 outer_side_key_types,
                 inner_side_key_types.clone(),
                 lookup_prefix_len,
-                source.epoch(),
+                query_epoch,
                 vec![],
                 table,
                 chunk_size,
diff --git a/src/batch/src/executor/join/hash_join.rs b/src/batch/src/executor/join/hash_join.rs
index 3bb97307b53a6..026f03fb65deb 100644
--- a/src/batch/src/executor/join/hash_join.rs
+++ b/src/batch/src/executor/join/hash_join.rs
@@ -22,7 +22,7 @@ use futures_async_stream::try_stream;
 use itertools::Itertools;
 use prost::Message;
 use risingwave_common::array::{Array, DataChunk, RowRef};
-use risingwave_common::buffer::{Bitmap, BitmapBuilder};
+use risingwave_common::bitmap::{Bitmap, BitmapBuilder};
 use risingwave_common::catalog::Schema;
 use risingwave_common::hash::{HashKey, HashKeyDispatcher, PrecomputedBuildHasher};
 use risingwave_common::memory::{MemoryContext, MonitoredGlobalAlloc};
@@ -907,7 +907,6 @@ impl<K: HashKey> HashJoinExecutor<K> {
                 if !visible {
                     continue;
                 }
-                non_equi_state.found_matched = false;
                 if let Some(first_matched_build_row_id) = hash_map.get(probe_key) {
                     non_equi_state
                         .first_output_row_id
@@ -2018,13 +2017,17 @@ impl DataChunkMutator {
                 new_visibility.set(row_id, true);
             }
         }
-        if !has_more_output_rows && !*found_non_null {
-            new_visibility.set(start_row_id, true);
+        if !has_more_output_rows {
+            if !*found_non_null {
+                new_visibility.set(start_row_id, true);
+            }
+            *found_non_null = false;
         }
 
         first_output_row_ids.clear();
 
-        self.0.set_visibility(new_visibility.finish());
+        self.0
+            .set_visibility(new_visibility.finish() & self.0.visibility());
         self
     }
 
@@ -3316,14 +3319,15 @@ mod tests {
              3   5.0 3   4.0
              3   5.0 3   3.0
              4   1.0 4   0
-             4   1.0 4   0.5",
+             4   1.0 4   9.0",
         );
         let expect = DataChunk::from_pretty(
             "i   f   i   F
              1   3.5 1   5.5
              2   4.0 .   .
              3   5.0 .   .
-             3   5.0 .   .",
+             3   5.0 .   .
+             4   1.0 4   9.0",
         );
         let cond = TestFixture::create_cond();
         let mut state = LeftNonEquiJoinState {
@@ -3344,7 +3348,7 @@ mod tests {
             &expect
         ));
         assert_eq!(state.first_output_row_id, Vec::<usize>::new());
-        assert!(!state.found_matched);
+        assert!(state.found_matched);
 
         let chunk = DataChunk::from_pretty(
             "i   f   i   F
@@ -3404,7 +3408,7 @@ mod tests {
             &expect
         ));
         assert_eq!(state.first_output_row_id, Vec::<usize>::new());
-        assert!(state.found_matched);
+        assert!(!state.found_matched);
     }
 
     #[tokio::test]
diff --git a/src/batch/src/executor/join/local_lookup_join.rs b/src/batch/src/executor/join/local_lookup_join.rs
index a00092b36d11c..7fcaba71a9c3b 100644
--- a/src/batch/src/executor/join/local_lookup_join.rs
+++ b/src/batch/src/executor/join/local_lookup_join.rs
@@ -17,7 +17,7 @@ use std::marker::PhantomData;
 
 use anyhow::Context;
 use itertools::Itertools;
-use risingwave_common::buffer::BitmapBuilder;
+use risingwave_common::bitmap::BitmapBuilder;
 use risingwave_common::catalog::{ColumnDesc, Field, Schema};
 use risingwave_common::hash::table_distribution::TableDistribution;
 use risingwave_common::hash::{
@@ -42,8 +42,8 @@ use risingwave_pb::plan_common::StorageTableDesc;
 
 use crate::error::Result;
 use crate::executor::{
-    BoxedDataChunkStream, BoxedExecutor, BoxedExecutorBuilder, DummyExecutor, Executor,
-    ExecutorBuilder, JoinType, LookupJoinBase,
+    unix_timestamp_sec_to_epoch, AsOf, BoxedDataChunkStream, BoxedExecutor, BoxedExecutorBuilder,
+    DummyExecutor, Executor, ExecutorBuilder, JoinType, LookupJoinBase,
 };
 use crate::task::{BatchTaskContext, ShutdownToken, TaskId};
 
@@ -66,6 +66,7 @@ struct InnerSideExecutorBuilder<C> {
     chunk_size: usize,
     shutdown_rx: ShutdownToken,
     next_stage_id: usize,
+    as_of: Option<AsOf>,
 }
 
 /// Used to build the executor for the inner side
@@ -108,6 +109,7 @@ impl<C: BatchTaskContext> InnerSideExecutorBuilder<C> {
             ordered: false,
             vnode_bitmap: Some(vnode_bitmap.finish().to_protobuf()),
             limit: None,
+            as_of: self.as_of.as_ref().map(Into::into),
         });
 
         Ok(row_seq_scan_node)
@@ -163,7 +165,7 @@ impl<C: BatchTaskContext> LookupExecutorBuilder for InnerSideExecutorBuilder<C>
         self.worker_slot_to_scan_range_mapping = HashMap::new();
     }
 
-    /// Adds the scan range made from the given `kwy_scalar_impls` into the parallel unit id
+    /// Adds the scan range made from the given `kwy_scalar_impls` into the worker slot id
     /// hash map, along with the scan range's virtual node.
     async fn add_scan_range(&mut self, key_datums: Vec<Datum>) -> Result<()> {
         let mut scan_range = ScanRange::full_table_scan();
@@ -296,6 +298,24 @@ impl BoxedExecutorBuilder for LocalLookupJoinExecutorBuilder {
             source.plan_node().get_node_body().unwrap(),
             NodeBody::LocalLookupJoin
         )?;
+        // as_of takes precedence
+        let as_of = lookup_join_node
+            .as_of
+            .as_ref()
+            .map(AsOf::try_from)
+            .transpose()?;
+        let query_epoch = as_of
+            .as_ref()
+            .map(|a| {
+                let epoch = unix_timestamp_sec_to_epoch(a.timestamp).0;
+                tracing::debug!(epoch, "time travel");
+                risingwave_pb::common::BatchQueryEpoch {
+                    epoch: Some(risingwave_pb::common::batch_query_epoch::Epoch::TimeTravel(
+                        epoch,
+                    )),
+                }
+            })
+            .unwrap_or_else(|| source.epoch());
 
         let join_type = JoinType::from_prost(lookup_join_node.get_join_type()?);
         let condition = match lookup_join_node.get_condition() {
@@ -383,7 +403,7 @@ impl BoxedExecutorBuilder for LocalLookupJoinExecutorBuilder {
         let worker_slot_mapping: HashMap<WorkerSlotId, WorkerNode> = worker_nodes
             .iter()
             .flat_map(|worker| {
-                (0..(worker.parallel_units.len()))
+                (0..(worker.parallelism()))
                     .map(|i| (WorkerSlotId::new(worker.id, i), worker.clone()))
             })
             .collect();
@@ -402,12 +422,13 @@ impl BoxedExecutorBuilder for LocalLookupJoinExecutorBuilder {
             lookup_prefix_len,
             context: source.context().clone(),
             task_id: source.task_id.clone(),
-            epoch: source.epoch(),
+            epoch: query_epoch,
             worker_slot_to_scan_range_mapping: HashMap::new(),
             chunk_size,
             shutdown_rx: source.shutdown_rx.clone(),
             next_stage_id: 0,
             worker_slot_mapping,
+            as_of,
         };
 
         let identity = source.plan_node().get_identity().clone();
diff --git a/src/batch/src/executor/join/mod.rs b/src/batch/src/executor/join/mod.rs
index 77210e3e6e25a..cf2388314d8f6 100644
--- a/src/batch/src/executor/join/mod.rs
+++ b/src/batch/src/executor/join/mod.rs
@@ -178,7 +178,7 @@ fn convert_row_to_chunk(
 mod tests {
 
     use risingwave_common::array::{Array, ArrayBuilder, DataChunk, PrimitiveArrayBuilder};
-    use risingwave_common::buffer::Bitmap;
+    use risingwave_common::bitmap::Bitmap;
     use risingwave_common::catalog::{Field, Schema};
     use risingwave_common::row::Row;
     use risingwave_common::types::{DataType, ScalarRefImpl};
diff --git a/src/batch/src/executor/join/nested_loop_join.rs b/src/batch/src/executor/join/nested_loop_join.rs
index 5d6d9b699b0e7..ae3e5b8fb4b35 100644
--- a/src/batch/src/executor/join/nested_loop_join.rs
+++ b/src/batch/src/executor/join/nested_loop_join.rs
@@ -15,7 +15,7 @@
 use futures_async_stream::try_stream;
 use risingwave_common::array::data_chunk_iter::RowRef;
 use risingwave_common::array::{Array, DataChunk};
-use risingwave_common::buffer::BitmapBuilder;
+use risingwave_common::bitmap::BitmapBuilder;
 use risingwave_common::catalog::Schema;
 use risingwave_common::memory::MemoryContext;
 use risingwave_common::row::{repeat_n, RowExt};
diff --git a/src/batch/src/executor/limit.rs b/src/batch/src/executor/limit.rs
index 8c02ed082f2b2..35e7a096714e2 100644
--- a/src/batch/src/executor/limit.rs
+++ b/src/batch/src/executor/limit.rs
@@ -17,7 +17,7 @@ use std::cmp::min;
 use futures_async_stream::try_stream;
 use itertools::Itertools;
 use risingwave_common::array::DataChunk;
-use risingwave_common::buffer::Bitmap;
+use risingwave_common::bitmap::Bitmap;
 use risingwave_common::catalog::Schema;
 use risingwave_pb::batch_plan::plan_node::NodeBody;
 
diff --git a/src/batch/src/executor/log_row_seq_scan.rs b/src/batch/src/executor/log_row_seq_scan.rs
index c46de5b3a2944..3614673ee941e 100644
--- a/src/batch/src/executor/log_row_seq_scan.rs
+++ b/src/batch/src/executor/log_row_seq_scan.rs
@@ -12,24 +12,23 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::ops::{Bound, Deref};
+use std::ops::Deref;
 use std::sync::Arc;
 
 use futures::prelude::stream::StreamExt;
 use futures_async_stream::try_stream;
 use futures_util::pin_mut;
-use itertools::Itertools;
 use prometheus::Histogram;
-use risingwave_common::array::DataChunk;
-use risingwave_common::buffer::Bitmap;
+use risingwave_common::array::{DataChunk, Op};
+use risingwave_common::bitmap::Bitmap;
 use risingwave_common::catalog::{ColumnId, Field, Schema};
-use risingwave_common::row::{OwnedRow, Row};
+use risingwave_common::row::{Row, RowExt};
 use risingwave_common::types::ScalarImpl;
 use risingwave_pb::batch_plan::plan_node::NodeBody;
-use risingwave_pb::common::BatchQueryEpoch;
+use risingwave_pb::common::{batch_query_epoch, BatchQueryEpoch};
 use risingwave_pb::plan_common::StorageTableDesc;
 use risingwave_storage::table::batch_table::storage_table::StorageTable;
-use risingwave_storage::table::{collect_data_chunk, KeyedRow, TableDistribution};
+use risingwave_storage::table::{collect_data_chunk, TableDistribution};
 use risingwave_storage::{dispatch_state_store, StateStore};
 
 use super::{BoxedDataChunkStream, BoxedExecutor, BoxedExecutorBuilder, Executor, ExecutorBuilder};
@@ -48,15 +47,15 @@ pub struct LogRowSeqScanExecutor<S: StateStore> {
     metrics: Option<BatchMetricsWithTaskLabels>,
 
     table: StorageTable<S>,
-    old_epoch: BatchQueryEpoch,
-    new_epoch: BatchQueryEpoch,
+    old_epoch: u64,
+    new_epoch: u64,
 }
 
 impl<S: StateStore> LogRowSeqScanExecutor<S> {
     pub fn new(
         table: StorageTable<S>,
-        old_epoch: BatchQueryEpoch,
-        new_epoch: BatchQueryEpoch,
+        old_epoch: u64,
+        new_epoch: u64,
         chunk_size: usize,
         identity: String,
         metrics: Option<BatchMetricsWithTaskLabels>,
@@ -113,12 +112,26 @@ impl BoxedExecutorBuilder for LogStoreRowSeqScanExecutorBuilder {
         let chunk_size = source.context.get_config().developer.chunk_size as u32;
         let metrics = source.context().batch_metrics();
 
+        let Some(BatchQueryEpoch {
+            epoch: Some(batch_query_epoch::Epoch::Committed(old_epoch)),
+        }) = &log_store_seq_scan_node.old_epoch
+        else {
+            unreachable!("invalid old epoch: {:?}", log_store_seq_scan_node.old_epoch)
+        };
+
+        let Some(BatchQueryEpoch {
+            epoch: Some(batch_query_epoch::Epoch::Committed(new_epoch)),
+        }) = &log_store_seq_scan_node.new_epoch
+        else {
+            unreachable!("invalid new epoch: {:?}", log_store_seq_scan_node.new_epoch)
+        };
+
         dispatch_state_store!(source.context().state_store(), state_store, {
             let table = StorageTable::new_partial(state_store, column_ids, vnodes, table_desc);
             Ok(Box::new(LogRowSeqScanExecutor::new(
                 table,
-                log_store_seq_scan_node.old_epoch.clone().unwrap(),
-                log_store_seq_scan_node.new_epoch.clone().unwrap(),
+                *old_epoch,
+                *new_epoch,
                 chunk_size as usize,
                 source.plan_node().get_identity().clone(),
                 metrics,
@@ -166,8 +179,8 @@ impl<S: StateStore> LogRowSeqScanExecutor<S> {
         //       it can consume too much memory if there're too many ranges.
         let stream = Self::execute_range(
             table.clone(),
-            old_epoch.clone(),
-            new_epoch.clone(),
+            old_epoch,
+            new_epoch,
             chunk_size,
             histogram.clone(),
             Arc::new(schema.clone()),
@@ -182,51 +195,35 @@ impl<S: StateStore> LogRowSeqScanExecutor<S> {
     #[try_stream(ok = DataChunk, error = BatchError)]
     async fn execute_range(
         table: Arc<StorageTable<S>>,
-        old_epoch: BatchQueryEpoch,
-        new_epoch: BatchQueryEpoch,
+        old_epoch: u64,
+        new_epoch: u64,
         chunk_size: usize,
         histogram: Option<impl Deref<Target = Histogram>>,
         schema: Arc<Schema>,
     ) {
-        let pk_prefix = OwnedRow::default();
-
-        let order_type = table.pk_serializer().get_order_types()[pk_prefix.len()];
         // Range Scan.
         let iter = table
-            .batch_iter_log_with_pk_bounds(
-                old_epoch.into(),
-                new_epoch.into(),
-                &pk_prefix,
-                (
-                    if order_type.nulls_are_first() {
-                        // `NULL`s are at the start bound side, we should exclude them to meet SQL semantics.
-                        Bound::Excluded(OwnedRow::new(vec![None]))
-                    } else {
-                        // Both start and end are unbounded, so we need to select all rows.
-                        Bound::Unbounded
-                    },
-                    if order_type.nulls_are_last() {
-                        // `NULL`s are at the end bound side, we should exclude them to meet SQL semantics.
-                        Bound::Excluded(OwnedRow::new(vec![None]))
-                    } else {
-                        // Both start and end are unbounded, so we need to select all rows.
-                        Bound::Unbounded
-                    },
-                ),
-            )
+            .batch_iter_log_with_pk_bounds(old_epoch, new_epoch)
             .await?
-            .map(|r| match r {
-                Ok((op, value)) => {
-                    let (k, row) = value.into_owned_row_key();
-                    // Todo! To avoid create a full row.
-                    let full_row = row
-                        .into_iter()
-                        .chain(vec![Some(ScalarImpl::Int16(op.to_i16()))])
-                        .collect_vec();
-                    let row = OwnedRow::new(full_row);
-                    Ok(KeyedRow::<_>::new(k, row))
-                }
-                Err(e) => Err(e),
+            .flat_map(|r| {
+                futures::stream::iter(std::iter::from_coroutine(
+                    #[coroutine]
+                    move || {
+                        match r {
+                            Ok(change_log_row) => {
+                                fn with_op(op: Op, row: impl Row) -> impl Row {
+                                    row.chain([Some(ScalarImpl::Int16(op.to_i16()))])
+                                }
+                                for (op, row) in change_log_row.into_op_value_iter() {
+                                    yield Ok(with_op(op, row));
+                                }
+                            }
+                            Err(e) => {
+                                yield Err(e);
+                            }
+                        };
+                    },
+                ))
             });
 
         pin_mut!(iter);
diff --git a/src/batch/src/executor/mod.rs b/src/batch/src/executor/mod.rs
index c19bc06c141b9..3a64901c64a04 100644
--- a/src/batch/src/executor/mod.rs
+++ b/src/batch/src/executor/mod.rs
@@ -33,6 +33,7 @@ mod order_by;
 mod project;
 mod project_set;
 mod row_seq_scan;
+mod s3_file_scan;
 mod sort_agg;
 mod sort_over_window;
 mod source;
@@ -86,6 +87,7 @@ pub use values::*;
 use self::log_row_seq_scan::LogStoreRowSeqScanExecutorBuilder;
 use self::test_utils::{BlockExecutorBuilder, BusyLoopExecutorBuilder};
 use crate::error::Result;
+use crate::executor::s3_file_scan::FileScanExecutorBuilder;
 use crate::executor::sys_row_seq_scan::SysRowSeqScanExecutorBuilder;
 use crate::task::{BatchTaskContext, ShutdownToken, TaskId};
 
@@ -240,6 +242,7 @@ impl<'a, C: BatchTaskContext> ExecutorBuilder<'a, C> {
             NodeBody::Source => SourceExecutor,
             NodeBody::SortOverWindow => SortOverWindowExecutor,
             NodeBody::MaxOneRow => MaxOneRowExecutor,
+            NodeBody::FileScan => FileScanExecutorBuilder,
             // Follow NodeBody only used for test
             NodeBody::BlockExecutor => BlockExecutorBuilder,
             NodeBody::BusyLoopExecutor => BusyLoopExecutorBuilder,
diff --git a/src/batch/src/executor/order_by.rs b/src/batch/src/executor/order_by.rs
index 1f7cbaacde6bd..3f8c8e106c78f 100644
--- a/src/batch/src/executor/order_by.rs
+++ b/src/batch/src/executor/order_by.rs
@@ -677,11 +677,11 @@ mod tests {
                 DecimalArray::from_iter([None, Some((-3).into()), None, None, Some(7.into())])
                     .into_ref(),
                 DateArray::from_iter([
-                    Some(Date::with_days(123).unwrap()),
-                    Some(Date::with_days(789).unwrap()),
-                    Some(Date::with_days(456).unwrap()),
+                    Some(Date::with_days_since_ce(123).unwrap()),
+                    Some(Date::with_days_since_ce(789).unwrap()),
+                    Some(Date::with_days_since_ce(456).unwrap()),
                     None,
-                    Some(Date::with_days(345).unwrap()),
+                    Some(Date::with_days_since_ce(345).unwrap()),
                 ])
                 .into_ref(),
             ],
@@ -698,11 +698,11 @@ mod tests {
                 DecimalArray::from_iter([Some(7.into()), Some((-3).into()), None, None, None])
                     .into_ref(),
                 DateArray::from_iter([
-                    Some(Date::with_days(345).unwrap()),
-                    Some(Date::with_days(789).unwrap()),
+                    Some(Date::with_days_since_ce(345).unwrap()),
+                    Some(Date::with_days_since_ce(789).unwrap()),
                     None,
-                    Some(Date::with_days(123).unwrap()),
-                    Some(Date::with_days(456).unwrap()),
+                    Some(Date::with_days_since_ce(123).unwrap()),
+                    Some(Date::with_days_since_ce(456).unwrap()),
                 ])
                 .into_ref(),
             ],
diff --git a/src/batch/src/executor/row_seq_scan.rs b/src/batch/src/executor/row_seq_scan.rs
index 505702ffa021b..b8287147c6750 100644
--- a/src/batch/src/executor/row_seq_scan.rs
+++ b/src/batch/src/executor/row_seq_scan.rs
@@ -19,7 +19,7 @@ use futures_async_stream::try_stream;
 use itertools::Itertools;
 use prometheus::Histogram;
 use risingwave_common::array::DataChunk;
-use risingwave_common::buffer::Bitmap;
+use risingwave_common::bitmap::Bitmap;
 use risingwave_common::catalog::{ColumnId, Schema};
 use risingwave_common::row::{OwnedRow, Row};
 use risingwave_common::types::{DataType, Datum};
@@ -28,7 +28,8 @@ use risingwave_common::util::value_encoding::deserialize_datum;
 use risingwave_pb::batch_plan::plan_node::NodeBody;
 use risingwave_pb::batch_plan::{scan_range, PbScanRange};
 use risingwave_pb::common::BatchQueryEpoch;
-use risingwave_pb::plan_common::StorageTableDesc;
+use risingwave_pb::plan_common::as_of::AsOfType;
+use risingwave_pb::plan_common::{as_of, PbAsOf, StorageTableDesc};
 use risingwave_storage::store::PrefetchOptions;
 use risingwave_storage::table::batch_table::storage_table::StorageTable;
 use risingwave_storage::table::TableDistribution;
@@ -55,6 +56,7 @@ pub struct RowSeqScanExecutor<S: StateStore> {
     ordered: bool,
     epoch: BatchQueryEpoch,
     limit: Option<u64>,
+    as_of: Option<AsOf>,
 }
 
 /// Range for batch scan.
@@ -66,6 +68,36 @@ pub struct ScanRange {
     pub next_col_bounds: (Bound<Datum>, Bound<Datum>),
 }
 
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct AsOf {
+    pub timestamp: i64,
+}
+
+impl TryFrom<&PbAsOf> for AsOf {
+    type Error = BatchError;
+
+    fn try_from(pb: &PbAsOf) -> std::result::Result<Self, Self::Error> {
+        match pb.as_of_type.as_ref().unwrap() {
+            AsOfType::Timestamp(ts) => Ok(Self {
+                timestamp: ts.timestamp,
+            }),
+            AsOfType::ProcessTime(_) | AsOfType::Version(_) => Err(BatchError::TimeTravel(
+                anyhow::anyhow!("batch query does not support as of process time or version"),
+            )),
+        }
+    }
+}
+
+impl From<&AsOf> for PbAsOf {
+    fn from(v: &AsOf) -> Self {
+        PbAsOf {
+            as_of_type: Some(AsOfType::Timestamp(as_of::Timestamp {
+                timestamp: v.timestamp,
+            })),
+        }
+    }
+}
+
 impl ScanRange {
     /// Create a scan range from the prost representation.
     pub fn new(
@@ -134,6 +166,7 @@ impl<S: StateStore> RowSeqScanExecutor<S> {
         identity: String,
         limit: Option<u64>,
         metrics: Option<BatchMetricsWithTaskLabels>,
+        as_of: Option<AsOf>,
     ) -> Self {
         Self {
             chunk_size,
@@ -144,6 +177,7 @@ impl<S: StateStore> RowSeqScanExecutor<S> {
             ordered,
             epoch,
             limit,
+            as_of,
         }
     }
 }
@@ -205,6 +239,11 @@ impl BoxedExecutorBuilder for RowSeqScanExecutorBuilder {
 
         let epoch = source.epoch.clone();
         let limit = seq_scan_node.limit;
+        let as_of = seq_scan_node
+            .as_of
+            .as_ref()
+            .map(AsOf::try_from)
+            .transpose()?;
         let chunk_size = if let Some(limit) = seq_scan_node.limit {
             (limit as u32).min(source.context.get_config().developer.chunk_size as u32)
         } else {
@@ -223,6 +262,7 @@ impl BoxedExecutorBuilder for RowSeqScanExecutorBuilder {
                 source.plan_node().get_identity().clone(),
                 limit,
                 metrics,
+                as_of,
             )))
         })
     }
@@ -254,8 +294,21 @@ impl<S: StateStore> RowSeqScanExecutor<S> {
             ordered,
             epoch,
             limit,
+            as_of,
         } = *self;
         let table = Arc::new(table);
+        // as_of takes precedence
+        let query_epoch = as_of
+            .map(|a| {
+                let epoch = unix_timestamp_sec_to_epoch(a.timestamp).0;
+                tracing::debug!(epoch, identity, "time travel");
+                risingwave_pb::common::BatchQueryEpoch {
+                    epoch: Some(risingwave_pb::common::batch_query_epoch::Epoch::TimeTravel(
+                        epoch,
+                    )),
+                }
+            })
+            .unwrap_or_else(|| epoch);
 
         // Create collector.
         let histogram = metrics.as_ref().map(|metrics| {
@@ -288,7 +341,8 @@ impl<S: StateStore> RowSeqScanExecutor<S> {
         for point_get in point_gets {
             let table = table.clone();
             if let Some(row) =
-                Self::execute_point_get(table, point_get, epoch.clone(), histogram.clone()).await?
+                Self::execute_point_get(table, point_get, query_epoch.clone(), histogram.clone())
+                    .await?
             {
                 if let Some(chunk) = data_chunk_builder.append_one_row(row) {
                     returned += chunk.cardinality() as u64;
@@ -319,7 +373,7 @@ impl<S: StateStore> RowSeqScanExecutor<S> {
                 table.clone(),
                 range,
                 ordered,
-                epoch.clone(),
+                query_epoch.clone(),
                 chunk_size,
                 limit,
                 histogram.clone(),
@@ -442,3 +496,10 @@ impl<S: StateStore> RowSeqScanExecutor<S> {
         }
     }
 }
+
+pub fn unix_timestamp_sec_to_epoch(ts: i64) -> risingwave_common::util::epoch::Epoch {
+    let ts = ts.checked_add(1).unwrap();
+    risingwave_common::util::epoch::Epoch::from_unix_millis_or_earliest(
+        u64::try_from(ts).unwrap_or(0).checked_mul(1000).unwrap(),
+    )
+}
diff --git a/src/batch/src/executor/s3_file_scan.rs b/src/batch/src/executor/s3_file_scan.rs
new file mode 100644
index 0000000000000..5096d1d625fa1
--- /dev/null
+++ b/src/batch/src/executor/s3_file_scan.rs
@@ -0,0 +1,150 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use anyhow::anyhow;
+use futures_async_stream::try_stream;
+use futures_util::stream::StreamExt;
+use parquet::arrow::ProjectionMask;
+use risingwave_common::array::arrow::IcebergArrowConvert;
+use risingwave_common::catalog::{Field, Schema};
+use risingwave_connector::source::iceberg::parquet_file_reader::create_parquet_stream_builder;
+use risingwave_pb::batch_plan::file_scan_node;
+use risingwave_pb::batch_plan::file_scan_node::StorageType;
+use risingwave_pb::batch_plan::plan_node::NodeBody;
+
+use crate::error::BatchError;
+use crate::executor::{BoxedExecutor, BoxedExecutorBuilder, DataChunk, Executor, ExecutorBuilder};
+use crate::task::BatchTaskContext;
+
+#[derive(PartialEq, Debug)]
+pub enum FileFormat {
+    Parquet,
+}
+
+/// S3 file scan executor. Currently only support parquet file format.
+pub struct S3FileScanExecutor {
+    file_format: FileFormat,
+    file_location: Vec<String>,
+    s3_region: String,
+    s3_access_key: String,
+    s3_secret_key: String,
+    batch_size: usize,
+    schema: Schema,
+    identity: String,
+}
+
+impl Executor for S3FileScanExecutor {
+    fn schema(&self) -> &risingwave_common::catalog::Schema {
+        &self.schema
+    }
+
+    fn identity(&self) -> &str {
+        &self.identity
+    }
+
+    fn execute(self: Box<Self>) -> super::BoxedDataChunkStream {
+        self.do_execute().boxed()
+    }
+}
+
+impl S3FileScanExecutor {
+    pub fn new(
+        file_format: FileFormat,
+        file_location: Vec<String>,
+        s3_region: String,
+        s3_access_key: String,
+        s3_secret_key: String,
+        batch_size: usize,
+        schema: Schema,
+        identity: String,
+    ) -> Self {
+        Self {
+            file_format,
+            file_location,
+            s3_region,
+            s3_access_key,
+            s3_secret_key,
+            batch_size,
+            schema,
+            identity,
+        }
+    }
+
+    #[try_stream(ok = DataChunk, error = BatchError)]
+    async fn do_execute(self: Box<Self>) {
+        assert_eq!(self.file_format, FileFormat::Parquet);
+        for file in self.file_location {
+            let mut batch_stream_builder = create_parquet_stream_builder(
+                self.s3_region.clone(),
+                self.s3_access_key.clone(),
+                self.s3_secret_key.clone(),
+                file,
+            )
+            .await?;
+
+            let arrow_schema = batch_stream_builder.schema();
+            assert_eq!(arrow_schema.fields.len(), self.schema.fields.len());
+            for (field, arrow_field) in self.schema.fields.iter().zip(arrow_schema.fields.iter()) {
+                assert_eq!(*field.name, *arrow_field.name());
+            }
+
+            batch_stream_builder = batch_stream_builder.with_projection(ProjectionMask::all());
+
+            batch_stream_builder = batch_stream_builder.with_batch_size(self.batch_size);
+
+            let record_batch_stream = batch_stream_builder
+                .build()
+                .map_err(|e| anyhow!(e).context("fail to build arrow stream builder"))?;
+
+            #[for_await]
+            for record_batch in record_batch_stream {
+                let record_batch = record_batch.map_err(BatchError::Parquet)?;
+                let chunk = IcebergArrowConvert.chunk_from_record_batch(&record_batch)?;
+                debug_assert_eq!(chunk.data_types(), self.schema.data_types());
+                yield chunk;
+            }
+        }
+    }
+}
+
+pub struct FileScanExecutorBuilder {}
+
+#[async_trait::async_trait]
+impl BoxedExecutorBuilder for FileScanExecutorBuilder {
+    async fn new_boxed_executor<C: BatchTaskContext>(
+        source: &ExecutorBuilder<'_, C>,
+        _inputs: Vec<BoxedExecutor>,
+    ) -> crate::error::Result<BoxedExecutor> {
+        let file_scan_node = try_match_expand!(
+            source.plan_node().get_node_body().unwrap(),
+            NodeBody::FileScan
+        )?;
+
+        assert_eq!(file_scan_node.storage_type, StorageType::S3 as i32);
+
+        Ok(Box::new(S3FileScanExecutor::new(
+            match file_scan_node::FileFormat::try_from(file_scan_node.file_format).unwrap() {
+                file_scan_node::FileFormat::Parquet => FileFormat::Parquet,
+                file_scan_node::FileFormat::Unspecified => unreachable!(),
+            },
+            file_scan_node.file_location.clone(),
+            file_scan_node.s3_region.clone(),
+            file_scan_node.s3_access_key.clone(),
+            file_scan_node.s3_secret_key.clone(),
+            source.context.get_config().developer.chunk_size,
+            Schema::from_iter(file_scan_node.columns.iter().map(Field::from)),
+            source.plan_node().get_identity().clone(),
+        )))
+    }
+}
diff --git a/src/batch/src/executor/source.rs b/src/batch/src/executor/source.rs
index c71f94ae36b09..e01ee9e4b7dee 100644
--- a/src/batch/src/executor/source.rs
+++ b/src/batch/src/executor/source.rs
@@ -27,13 +27,12 @@ use risingwave_connector::source::reader::reader::SourceReader;
 use risingwave_connector::source::{
     ConnectorProperties, SourceColumnDesc, SourceContext, SourceCtrlOpts, SplitImpl, SplitMetaData,
 };
+use risingwave_connector::WithOptionsSecResolved;
 use risingwave_pb::batch_plan::plan_node::NodeBody;
 
 use super::Executor;
 use crate::error::{BatchError, Result};
-use crate::executor::{
-    BoxedExecutor, BoxedExecutorBuilder, ExecutorBuilder, FileSelector, IcebergScanExecutor,
-};
+use crate::executor::{BoxedExecutor, BoxedExecutorBuilder, ExecutorBuilder, IcebergScanExecutor};
 use crate::task::BatchTaskContext;
 
 pub struct SourceExecutor {
@@ -64,12 +63,15 @@ impl BoxedExecutorBuilder for SourceExecutor {
         )?;
 
         // prepare connector source
-        let source_props = source_node.with_properties.clone();
-        let config =
-            ConnectorProperties::extract(source_props, false).map_err(BatchError::connector)?;
+        let options_with_secret = WithOptionsSecResolved::new(
+            source_node.with_properties.clone(),
+            source_node.secret_refs.clone(),
+        );
+        let config = ConnectorProperties::extract(options_with_secret.clone(), false)
+            .map_err(BatchError::connector)?;
 
         let info = source_node.get_info().unwrap();
-        let parser_config = SpecificParserConfig::new(info, &source_node.with_properties)?;
+        let parser_config = SpecificParserConfig::new(info, &options_with_secret)?;
 
         let columns: Vec<_> = source_node
             .columns
@@ -109,7 +111,8 @@ impl BoxedExecutorBuilder for SourceExecutor {
                 Ok(Box::new(IcebergScanExecutor::new(
                     iceberg_properties.to_iceberg_config(),
                     Some(split.snapshot_id),
-                    FileSelector::FileList(split.files),
+                    split.table_meta.deserialize(),
+                    split.files.into_iter().map(|x| x.deserialize()).collect(),
                     source.context.get_config().developer.chunk_size,
                     schema,
                     source.plan_node().get_identity().clone(),
diff --git a/src/batch/src/lib.rs b/src/batch/src/lib.rs
index 3a29e2a90b27e..414f27b33b4a7 100644
--- a/src/batch/src/lib.rs
+++ b/src/batch/src/lib.rs
@@ -28,9 +28,9 @@
 #![feature(allocator_api)]
 #![feature(impl_trait_in_assoc_type)]
 #![feature(assert_matches)]
-#![feature(lazy_cell)]
 #![feature(error_generic_member_access)]
 #![feature(map_try_insert)]
+#![feature(iter_from_coroutine)]
 
 pub mod error;
 pub mod exchange_source;
diff --git a/src/batch/src/task/consistent_hash_shuffle_channel.rs b/src/batch/src/task/consistent_hash_shuffle_channel.rs
index 1181654f41249..ad0fdbaa8b70a 100644
--- a/src/batch/src/task/consistent_hash_shuffle_channel.rs
+++ b/src/batch/src/task/consistent_hash_shuffle_channel.rs
@@ -18,7 +18,7 @@ use std::sync::Arc;
 use anyhow::anyhow;
 use itertools::Itertools;
 use risingwave_common::array::DataChunk;
-use risingwave_common::buffer::Bitmap;
+use risingwave_common::bitmap::Bitmap;
 use risingwave_common::hash::VirtualNode;
 use risingwave_pb::batch_plan::exchange_info::ConsistentHashInfo;
 use risingwave_pb::batch_plan::*;
diff --git a/src/batch/src/task/env.rs b/src/batch/src/task/env.rs
index ecb7a3a8d3ebb..2f50471a91f80 100644
--- a/src/batch/src/task/env.rs
+++ b/src/batch/src/task/env.rs
@@ -52,7 +52,7 @@ pub struct BatchEnvironment {
     /// Executor level metrics.
     executor_metrics: Arc<BatchExecutorMetrics>,
 
-    /// Compute client pool for grpc exchange.
+    /// Compute client pool for batch gRPC exchange.
     client_pool: ComputeClientPoolRef,
 
     /// Manages dml information.
@@ -112,14 +112,14 @@ impl BatchEnvironment {
                 BatchManagerMetrics::for_test(),
                 u64::MAX,
             )),
-            server_addr: "127.0.0.1:5688".parse().unwrap(),
+            server_addr: "127.0.0.1:2333".parse().unwrap(),
             config: Arc::new(BatchConfig::default()),
             worker_id: WorkerNodeId::default(),
             state_store: StateStoreImpl::shared_in_memory_store(Arc::new(
                 MonitoredStorageMetrics::unused(),
             )),
             task_metrics: Arc::new(BatchTaskMetrics::for_test()),
-            client_pool: Arc::new(ComputeClientPool::default()),
+            client_pool: Arc::new(ComputeClientPool::for_test()),
             dml_manager: Arc::new(DmlManager::for_test()),
             source_metrics: Arc::new(SourceMetrics::default()),
             executor_metrics: Arc::new(BatchExecutorMetrics::for_test()),
diff --git a/src/batch/src/task/hash_shuffle_channel.rs b/src/batch/src/task/hash_shuffle_channel.rs
index 626324e6f3456..ccea1529b40f6 100644
--- a/src/batch/src/task/hash_shuffle_channel.rs
+++ b/src/batch/src/task/hash_shuffle_channel.rs
@@ -17,7 +17,7 @@ use std::sync::Arc;
 
 use anyhow::anyhow;
 use risingwave_common::array::DataChunk;
-use risingwave_common::buffer::Bitmap;
+use risingwave_common::bitmap::Bitmap;
 use risingwave_common::util::hash_util::Crc32FastBuilder;
 use risingwave_pb::batch_plan::exchange_info::HashInfo;
 use risingwave_pb::batch_plan::*;
diff --git a/src/batch/src/worker_manager/worker_node_manager.rs b/src/batch/src/worker_manager/worker_node_manager.rs
index 8b7dcc42b565a..c4f4829c36110 100644
--- a/src/batch/src/worker_manager/worker_node_manager.rs
+++ b/src/batch/src/worker_manager/worker_node_manager.rs
@@ -18,6 +18,7 @@ use std::time::Duration;
 
 use rand::seq::SliceRandom;
 use risingwave_common::bail;
+use risingwave_common::catalog::OBJECT_ID_PLACEHOLDER;
 use risingwave_common::hash::{WorkerSlotId, WorkerSlotMapping};
 use risingwave_common::vnode_mapping::vnode_placement::place_vnode;
 use risingwave_pb::common::{WorkerNode, WorkerType};
@@ -156,19 +157,12 @@ impl WorkerNodeManager {
 
         let guard = self.inner.read().unwrap();
 
-        let worker_slot_index: HashMap<_, _> = guard
-            .worker_nodes
-            .iter()
-            .flat_map(|worker| {
-                (0..worker.parallel_units.len())
-                    .map(move |i| (WorkerSlotId::new(worker.id, i), worker))
-            })
-            .collect();
+        let worker_index: HashMap<_, _> = guard.worker_nodes.iter().map(|w| (w.id, w)).collect();
 
         let mut workers = Vec::with_capacity(worker_slot_ids.len());
 
         for worker_slot_id in worker_slot_ids {
-            match worker_slot_index.get(worker_slot_id) {
+            match worker_index.get(&worker_slot_id.worker_id()) {
                 Some(worker) => workers.push((*worker).clone()),
                 None => bail!(
                     "No worker node found for worker slot id: {}",
@@ -220,10 +214,20 @@ impl WorkerNodeManager {
 
     pub fn remove_streaming_fragment_mapping(&self, fragment_id: &FragmentId) {
         let mut guard = self.inner.write().unwrap();
-        guard
-            .streaming_fragment_vnode_mapping
-            .remove(fragment_id)
-            .unwrap();
+
+        let res = guard.streaming_fragment_vnode_mapping.remove(fragment_id);
+        match &res {
+            Some(_) => {}
+            None if OBJECT_ID_PLACEHOLDER == *fragment_id => {
+                // Do nothing for placeholder fragment.
+            }
+            None => {
+                panic!(
+                    "Streaming vnode mapping not found for fragment_id: {}",
+                    fragment_id
+                )
+            }
+        };
     }
 
     /// Returns fragment's vnode mapping for serving.
@@ -335,10 +339,7 @@ impl WorkerNodeSelector {
         } else {
             self.apply_worker_node_mask(self.manager.list_serving_worker_nodes())
         };
-        worker_nodes
-            .iter()
-            .map(|node| node.parallel_units.len())
-            .sum()
+        worker_nodes.iter().map(|node| node.parallelism()).sum()
     }
 
     pub fn fragment_mapping(&self, fragment_id: FragmentId) -> Result<WorkerSlotMapping> {
@@ -424,7 +425,7 @@ mod tests {
                 r#type: WorkerType::ComputeNode as i32,
                 host: Some(HostAddr::try_from("127.0.0.1:1234").unwrap().to_protobuf()),
                 state: worker_node::State::Running as i32,
-                parallel_units: vec![],
+                parallelism: 0,
                 property: Some(Property {
                     is_unschedulable: false,
                     is_serving: true,
@@ -438,7 +439,7 @@ mod tests {
                 r#type: WorkerType::ComputeNode as i32,
                 host: Some(HostAddr::try_from("127.0.0.1:1235").unwrap().to_protobuf()),
                 state: worker_node::State::Running as i32,
-                parallel_units: vec![],
+                parallelism: 0,
                 property: Some(Property {
                     is_unschedulable: false,
                     is_serving: true,
diff --git a/src/bench/Cargo.toml b/src/bench/Cargo.toml
index 32dcdaa2a5df6..d451ef46ef838 100644
--- a/src/bench/Cargo.toml
+++ b/src/bench/Cargo.toml
@@ -25,7 +25,11 @@ itertools = { workspace = true }
 libc = "0.2"
 opentelemetry = { workspace = true, optional = true }
 parking_lot = { workspace = true }
-plotlib = "0.5.1"
+plotters = { version = "0.3.5", default-features = false, features = [
+    "svg_backend",
+    "line_series",
+    "point_series"
+] }
 prometheus = { version = "0.13", features = ["process"] }
 rand = { workspace = true }
 risingwave_common = { workspace = true }
diff --git a/src/bench/sink_bench/main.rs b/src/bench/sink_bench/main.rs
index 189afdf6e83d4..0aa70ad00c364 100644
--- a/src/bench/sink_bench/main.rs
+++ b/src/bench/sink_bench/main.rs
@@ -31,11 +31,13 @@ use futures::stream::select_with_strategy;
 use futures::{FutureExt, StreamExt, TryStreamExt};
 use futures_async_stream::try_stream;
 use itertools::Itertools;
-use plotlib::page::Page;
-use plotlib::repr::Plot;
-use plotlib::style::{LineJoin, LineStyle};
-use plotlib::view::ContinuousView;
-use risingwave_common::buffer::Bitmap;
+use plotters::backend::SVGBackend;
+use plotters::chart::ChartBuilder;
+use plotters::drawing::IntoDrawingArea;
+use plotters::element::{Circle, EmptyElement};
+use plotters::series::{LineSeries, PointSeries};
+use plotters::style::{IntoFont, RED, WHITE};
+use risingwave_common::bitmap::Bitmap;
 use risingwave_common::catalog::ColumnId;
 use risingwave_connector::dispatch_sink;
 use risingwave_connector::parser::{
@@ -215,34 +217,48 @@ impl ThroughputMetric {
         println!("p90: {:?} rows/s ", p90);
         println!("p95: {:?} rows/s ", p95);
         println!("p99: {:?} rows/s ", p99);
-        let draw_vec: Vec<(f64, f64)> = throughput_vec
+        let draw_vec: Vec<(f32, f32)> = throughput_vec
             .iter()
             .enumerate()
             .map(|(index, &value)| {
                 (
-                    (index as f64) * (THROUGHPUT_METRIC_RECORD_INTERVAL as f64 / 1000_f64),
-                    value as f64,
+                    (index as f32) * (THROUGHPUT_METRIC_RECORD_INTERVAL as f32 / 1000_f32),
+                    value as f32,
                 )
             })
             .collect();
 
-        let s1: Plot = Plot::new(draw_vec).line_style(
-            LineStyle::new()
-                .colour("burlywood")
-                .linejoin(LineJoin::Round),
-        );
-
-        let v = ContinuousView::new()
-            .add(s1)
-            .x_range(0.0, BENCH_TIME as f64)
-            .y_range(
-                **throughput_vec_sorted.first().unwrap() as f64,
-                **throughput_vec_sorted.last().unwrap() as f64,
+        let root = SVGBackend::new("throughput.svg", (640, 480)).into_drawing_area();
+        root.fill(&WHITE).unwrap();
+        let root = root.margin(10, 10, 10, 10);
+        let mut chart = ChartBuilder::on(&root)
+            .caption("Throughput Sink", ("sans-serif", 40).into_font())
+            .x_label_area_size(20)
+            .y_label_area_size(40)
+            .build_cartesian_2d(
+                0.0..BENCH_TIME as f32,
+                **throughput_vec_sorted.first().unwrap() as f32
+                    ..**throughput_vec_sorted.last().unwrap() as f32,
             )
-            .x_label("Time (s)")
-            .y_label("Throughput (rows/s)");
+            .unwrap();
 
-        Page::single(&v).save("throughput.svg").unwrap();
+        chart
+            .configure_mesh()
+            .x_labels(5)
+            .y_labels(5)
+            .y_label_formatter(&|x| format!("{:.0}", x))
+            .draw()
+            .unwrap();
+
+        chart
+            .draw_series(LineSeries::new(draw_vec.clone(), &RED))
+            .unwrap();
+        chart
+            .draw_series(PointSeries::of_element(draw_vec, 5, &RED, &|c, s, st| {
+                EmptyElement::at(c) + Circle::new((0, 0), s, st.filled())
+            }))
+            .unwrap();
+        root.present().unwrap();
 
         println!(
             "Throughput Sink: {:?}",
@@ -469,6 +485,7 @@ fn mock_from_legacy_type(
             format,
             encode: SinkEncode::Json,
             options: Default::default(),
+            secret_refs: Default::default(),
             key_encode: None,
         }))
     } else {
diff --git a/src/cmd/src/lib.rs b/src/cmd/src/lib.rs
index c6bf6c849fd39..9017f58606f7c 100644
--- a/src/cmd/src/lib.rs
+++ b/src/cmd/src/lib.rs
@@ -44,8 +44,7 @@ pub fn compute(opts: ComputeNodeOpts) -> ! {
 
 pub fn meta(opts: MetaNodeOpts) -> ! {
     init_risingwave_logger(LoggerSettings::from_opts(&opts));
-    // TODO(shutdown): pass the shutdown token
-    main_okk(|_| risingwave_meta_node::start(opts));
+    main_okk(|shutdown| risingwave_meta_node::start(opts, shutdown));
 }
 
 pub fn frontend(opts: FrontendOpts) -> ! {
@@ -55,12 +54,10 @@ pub fn frontend(opts: FrontendOpts) -> ! {
 
 pub fn compactor(opts: CompactorOpts) -> ! {
     init_risingwave_logger(LoggerSettings::from_opts(&opts));
-    // TODO(shutdown): pass the shutdown token
-    main_okk(|_| risingwave_compactor::start(opts));
+    main_okk(|shutdown| risingwave_compactor::start(opts, shutdown));
 }
 
 pub fn ctl(opts: CtlOpts) -> ! {
     init_risingwave_logger(LoggerSettings::new("ctl").stderr(true));
-    // TODO(shutdown): pass the shutdown token
-    main_okk(|_| risingwave_ctl::start(opts));
+    main_okk(|shutdown| risingwave_ctl::start(opts, shutdown));
 }
diff --git a/src/cmd_all/Cargo.toml b/src/cmd_all/Cargo.toml
index d3341b3137d43..aaa9f2886eaa8 100644
--- a/src/cmd_all/Cargo.toml
+++ b/src/cmd_all/Cargo.toml
@@ -11,11 +11,10 @@ repository = { workspace = true }
 default = ["rw-static-link"]
 rw-static-link = ["workspace-config/rw-static-link"]
 rw-dynamic-link = ["workspace-config/rw-dynamic-link"]
-all-udf = ["external-udf", "wasm-udf", "js-udf", "deno-udf", "python-udf"]
+all-udf = ["external-udf", "wasm-udf", "js-udf", "python-udf"]
 external-udf = ["risingwave_expr_impl/external-udf"]
 wasm-udf = ["risingwave_expr_impl/wasm-udf"]
 js-udf = ["risingwave_expr_impl/js-udf"]
-deno-udf = ["risingwave_expr_impl/deno-udf"]
 python-udf = ["risingwave_expr_impl/python-udf"]
 
 [package.metadata.cargo-machete]
diff --git a/src/cmd_all/scripts/e2e-full-standalone-demo.sh b/src/cmd_all/scripts/e2e-full-standalone-demo.sh
index 409c543e03afa..d3c8e10529f4c 100755
--- a/src/cmd_all/scripts/e2e-full-standalone-demo.sh
+++ b/src/cmd_all/scripts/e2e-full-standalone-demo.sh
@@ -61,13 +61,6 @@ STANDALONE_PID=$!
 
 sleep 15
 
-# FIXME: Integrate standalone into risedev, so we can reuse risedev-env functionality here.
-cat << EOF > "$RW_PREFIX"/config/risedev-env
-RW_META_ADDR="http://0.0.0.0:5690"
-RISEDEV_RW_FRONTEND_LISTEN_ADDRESS="0.0.0.0"
-RISEDEV_RW_FRONTEND_PORT="4566"
-EOF
-
 echo "--- Setting up table"
 ./risedev psql -c "
 CREATE TABLE t (v1 int);
diff --git a/src/cmd_all/src/bin/risingwave.rs b/src/cmd_all/src/bin/risingwave.rs
index 13c73217b77cc..c6ca4675bb814 100644
--- a/src/cmd_all/src/bin/risingwave.rs
+++ b/src/cmd_all/src/bin/risingwave.rs
@@ -230,8 +230,7 @@ fn standalone(opts: StandaloneOpts) -> ! {
         .with_target("risingwave_storage", Level::WARN)
         .with_thread_name(true);
     risingwave_rt::init_risingwave_logger(settings);
-    // TODO(shutdown): pass the shutdown token
-    risingwave_rt::main_okk(|_| risingwave_cmd_all::standalone(opts));
+    risingwave_rt::main_okk(|shutdown| risingwave_cmd_all::standalone(opts, shutdown));
 }
 
 /// For single node, the internals are just a config mapping from its
@@ -246,8 +245,7 @@ fn single_node(opts: SingleNodeOpts) -> ! {
         .with_target("risingwave_storage", Level::WARN)
         .with_thread_name(true);
     risingwave_rt::init_risingwave_logger(settings);
-    // TODO(shutdown): pass the shutdown token
-    risingwave_rt::main_okk(|_| risingwave_cmd_all::standalone(opts));
+    risingwave_rt::main_okk(|shutdown| risingwave_cmd_all::standalone(opts, shutdown));
 }
 
 #[cfg(test)]
diff --git a/src/cmd_all/src/lib.rs b/src/cmd_all/src/lib.rs
index a872d7de12b39..9b95af8dce4d3 100644
--- a/src/cmd_all/src/lib.rs
+++ b/src/cmd_all/src/lib.rs
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#![feature(lazy_cell)]
+#![feature(let_chains)]
 
 mod common;
 mod standalone;
diff --git a/src/cmd_all/src/standalone.rs b/src/cmd_all/src/standalone.rs
index 791921fc07d69..ceb890f4cb3af 100644
--- a/src/cmd_all/src/standalone.rs
+++ b/src/cmd_all/src/standalone.rs
@@ -12,16 +12,22 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::env;
+use std::fmt::Write;
+use std::future::Future;
+use std::path::Path;
+
 use clap::Parser;
 use risingwave_common::config::MetaBackend;
+use risingwave_common::util::env_var::env_var_is_true;
 use risingwave_common::util::meta_addr::MetaAddressStrategy;
+use risingwave_common::util::runtime::BackgroundShutdownRuntime;
 use risingwave_common::util::tokio_util::sync::CancellationToken;
 use risingwave_compactor::CompactorOpts;
 use risingwave_compute::ComputeNodeOpts;
 use risingwave_frontend::FrontendOpts;
 use risingwave_meta_node::MetaNodeOpts;
 use shell_words::split;
-use tokio::signal;
 
 use crate::common::osstrs;
 
@@ -173,9 +179,65 @@ pub fn parse_standalone_opt_args(opts: &StandaloneOpts) -> ParsedStandaloneOpts
     }
 }
 
+/// A service under standalone mode.
+struct Service {
+    name: &'static str,
+    runtime: BackgroundShutdownRuntime,
+    main_task: tokio::task::JoinHandle<()>,
+    shutdown: CancellationToken,
+}
+
+impl Service {
+    /// Spawn a new tokio runtime and start a service in it.
+    ///
+    /// By using a separate runtime, we get better isolation between services. For example,
+    ///
+    /// - The logs in the main runtime of each service can be distinguished by the thread name.
+    /// - Each service can be shutdown cleanly by shutting down its runtime.
+    fn spawn<F, Fut>(name: &'static str, f: F) -> Self
+    where
+        F: FnOnce(CancellationToken) -> Fut,
+        Fut: Future<Output = ()> + Send + 'static,
+    {
+        let runtime = tokio::runtime::Builder::new_multi_thread()
+            .thread_name(format!("rw-standalone-{name}"))
+            .enable_all()
+            .build()
+            .unwrap();
+        let shutdown = CancellationToken::new();
+        let main_task = runtime.spawn(f(shutdown.clone()));
+
+        Self {
+            name,
+            runtime: runtime.into(),
+            main_task,
+            shutdown,
+        }
+    }
+
+    /// Shutdown the service and the runtime gracefully.
+    ///
+    /// As long as the main task of the service is resolved after signaling `shutdown`,
+    /// the service is considered stopped and the runtime will be shutdown. This follows
+    /// the same convention as described in `risingwave_rt::main_okk`.
+    async fn shutdown(self) {
+        tracing::info!("stopping {} service...", self.name);
+
+        self.shutdown.cancel();
+        let _ = self.main_task.await;
+        drop(self.runtime); // shutdown in background
+
+        tracing::info!("{} service stopped", self.name);
+    }
+}
+
 /// For `standalone` mode, we can configure and start multiple services in one process.
 /// `standalone` mode is meant to be used by our cloud service and docker,
 /// where we can configure and start multiple services in one process.
+///
+/// Services are started in the order of `meta`, `compute`, `frontend`, then `compactor`.
+/// When the `shutdown` token is signaled, all services will be stopped gracefully in the
+/// reverse order.
 pub async fn standalone(
     ParsedStandaloneOpts {
         meta_opts,
@@ -183,59 +245,69 @@ pub async fn standalone(
         frontend_opts,
         compactor_opts,
     }: ParsedStandaloneOpts,
+    shutdown: CancellationToken,
 ) {
     tracing::info!("launching Risingwave in standalone mode");
 
-    // TODO(shutdown): use the real one passed-in
-    let shutdown = CancellationToken::new();
-
-    let mut is_in_memory = false;
-    if let Some(opts) = meta_opts {
-        is_in_memory = matches!(opts.backend, Some(MetaBackend::Mem));
+    let (meta, is_in_memory) = if let Some(opts) = meta_opts.clone() {
+        let is_in_memory = matches!(opts.backend, Some(MetaBackend::Mem));
         tracing::info!("starting meta-node thread with cli args: {:?}", opts);
-
-        let _meta_handle = tokio::spawn(async move {
-            let dangerous_max_idle_secs = opts.dangerous_max_idle_secs;
-            risingwave_meta_node::start(opts).await;
-            tracing::warn!("meta is stopped, shutdown all nodes");
-            if let Some(idle_exit_secs) = dangerous_max_idle_secs {
-                eprintln!("{}",
-                          console::style(format_args!(
-                              "RisingWave playground exited after being idle for {idle_exit_secs} seconds. Bye!"
-                          )).bold());
-                std::process::exit(0);
-            }
+        let service = Service::spawn("meta", |shutdown| {
+            risingwave_meta_node::start(opts, shutdown)
         });
+
         // wait for the service to be ready
         let mut tries = 0;
         while !risingwave_meta_node::is_server_started() {
             if tries % 50 == 0 {
                 tracing::info!("waiting for meta service to be ready...");
             }
+            if service.main_task.is_finished() {
+                tracing::error!("meta service failed to start, exiting...");
+                return;
+            }
             tries += 1;
             tokio::time::sleep(std::time::Duration::from_millis(100)).await;
         }
-    }
-    if let Some(opts) = compute_opts {
+
+        (Some(service), is_in_memory)
+    } else {
+        (None, false)
+    };
+
+    let compute = if let Some(opts) = compute_opts {
         tracing::info!("starting compute-node thread with cli args: {:?}", opts);
-        let shutdown = shutdown.clone();
-        let _compute_handle =
-            tokio::spawn(async move { risingwave_compute::start(opts, shutdown).await });
-    }
-    if let Some(opts) = frontend_opts.clone() {
+        let service = Service::spawn("compute", |shutdown| {
+            risingwave_compute::start(opts, shutdown)
+        });
+        Some(service)
+    } else {
+        None
+    };
+
+    let frontend = if let Some(opts) = frontend_opts.clone() {
         tracing::info!("starting frontend-node thread with cli args: {:?}", opts);
-        let shutdown = shutdown.clone();
-        let _frontend_handle =
-            tokio::spawn(async move { risingwave_frontend::start(opts, shutdown).await });
-    }
-    if let Some(opts) = compactor_opts {
+        let service = Service::spawn("frontend", |shutdown| {
+            risingwave_frontend::start(opts, shutdown)
+        });
+        Some(service)
+    } else {
+        None
+    };
+
+    let compactor = if let Some(opts) = compactor_opts {
         tracing::info!("starting compactor-node thread with cli args: {:?}", opts);
-        let _compactor_handle =
-            tokio::spawn(async move { risingwave_compactor::start(opts).await });
-    }
+        let service = Service::spawn("compactor", |shutdown| {
+            risingwave_compactor::start(opts, shutdown)
+        });
+        Some(service)
+    } else {
+        None
+    };
 
     // wait for log messages to be flushed
     tokio::time::sleep(std::time::Duration::from_millis(5000)).await;
+
     eprintln!("----------------------------------------");
     eprintln!("| RisingWave standalone mode is ready. |");
     eprintln!("----------------------------------------");
@@ -250,11 +322,23 @@ It SHOULD NEVER be used in benchmarks and production environment!!!"
             .bold()
         );
     }
-    if let Some(opts) = frontend_opts {
+
+    // This is a specialization of `generate_risedev_env` in `src/risedevtool/src/risedev_env.rs`.
+    let mut risedev_env = String::new();
+
+    if let Some(opts) = &frontend_opts {
         let host = opts.listen_addr.split(':').next().unwrap_or("localhost");
         let port = opts.listen_addr.split(':').last().unwrap_or("4566");
         let database = "dev";
         let user = "root";
+
+        writeln!(
+            risedev_env,
+            r#"RISEDEV_RW_FRONTEND_LISTEN_ADDRESS="{host}""#
+        )
+        .unwrap();
+        writeln!(risedev_env, r#"RISEDEV_RW_FRONTEND_PORT="{port}""#).unwrap();
+
         eprintln!();
         eprintln!("Connect to the RisingWave instance via psql:");
         eprintln!(
@@ -266,12 +350,43 @@ It SHOULD NEVER be used in benchmarks and production environment!!!"
         );
     }
 
-    // TODO: should we join all handles?
-    // Currently, not all services can be shutdown gracefully, just quit on Ctrl-C now.
-    // TODO(kwannoel): Why can't be shutdown gracefully? Is it that the service just does not
-    // support it?
-    signal::ctrl_c().await.unwrap();
-    tracing::info!("Ctrl+C received, now exiting");
+    if let Some(opts) = &meta_opts {
+        let meta_addr = &opts.listen_addr;
+        writeln!(risedev_env, r#"RW_META_ADDR="http://{meta_addr}""#).unwrap();
+    }
+
+    // Create the environment file when launched by RiseDev.
+    if env_var_is_true("RISEDEV") {
+        let env_path = Path::new(
+            &env::var("PREFIX_CONFIG").expect("env var `PREFIX_CONFIG` must be set by RiseDev"),
+        )
+        .join("risedev-env");
+        std::fs::write(env_path, risedev_env).unwrap();
+    }
+
+    let meta_stopped = meta
+        .as_ref()
+        .map(|m| m.shutdown.clone())
+        // If there's no meta service, use a dummy token which will never resolve.
+        .unwrap_or_else(CancellationToken::new)
+        .cancelled_owned();
+
+    // Wait for shutdown signals.
+    tokio::select! {
+        // Meta service stopped itself, typically due to leadership loss of idleness.
+        // Directly exit in this case.
+        _ = meta_stopped => {
+            tracing::info!("meta service is stopped, terminating...");
+        }
+
+        // Shutdown requested by the user.
+        _ = shutdown.cancelled() => {
+            for service in [compactor, frontend, compute, meta].into_iter().flatten() {
+                service.shutdown().await;
+            }
+            tracing::info!("all services stopped, bye");
+        }
+    }
 }
 
 #[cfg(test)]
@@ -291,17 +406,17 @@ mod test {
     fn test_parse_opt_args() {
         // Test parsing into standalone-level opts.
         let raw_opts = "
---compute-opts=--listen-addr 127.0.0.1:8000 --total-memory-bytes 34359738368 --parallelism 10
---meta-opts=--advertise-addr 127.0.0.1:9999 --data-directory \"some path with spaces\" --listen-addr 127.0.0.1:8001 --etcd-password 1234
---frontend-opts=--config-path=src/config/original.toml
+--compute-opts=--listen-addr 127.0.0.1:8000 --total-memory-bytes 34359738368 --parallelism 10 --temp-secret-file-dir ./compute/secrets/
+--meta-opts=--advertise-addr 127.0.0.1:9999 --data-directory \"some path with spaces\" --listen-addr 127.0.0.1:8001 --etcd-password 1234 --temp-secret-file-dir ./meta/secrets/
+--frontend-opts=--config-path=src/config/original.toml --temp-secret-file-dir ./frontend/secrets/
 --prometheus-listener-addr=127.0.0.1:1234
 --config-path=src/config/test.toml
 ";
         let actual = StandaloneOpts::parse_from(raw_opts.lines());
         let opts = StandaloneOpts {
-            compute_opts: Some("--listen-addr 127.0.0.1:8000 --total-memory-bytes 34359738368 --parallelism 10".into()),
-            meta_opts: Some("--advertise-addr 127.0.0.1:9999 --data-directory \"some path with spaces\" --listen-addr 127.0.0.1:8001 --etcd-password 1234".into()),
-            frontend_opts: Some("--config-path=src/config/original.toml".into()),
+            compute_opts: Some("--listen-addr 127.0.0.1:8000 --total-memory-bytes 34359738368 --parallelism 10 --temp-secret-file-dir ./compute/secrets/".into()),
+            meta_opts: Some("--advertise-addr 127.0.0.1:9999 --data-directory \"some path with spaces\" --listen-addr 127.0.0.1:8001 --etcd-password 1234 --temp-secret-file-dir ./meta/secrets/".into()),
+            frontend_opts: Some("--config-path=src/config/original.toml --temp-secret-file-dir ./frontend/secrets/".into()),
             compactor_opts: None,
             prometheus_listener_addr: Some("127.0.0.1:1234".into()),
             config_path: Some("src/config/test.toml".into()),
@@ -352,6 +467,7 @@ mod test {
                             heap_profiling_dir: None,
                             dangerous_max_idle_secs: None,
                             connector_rpc_endpoint: None,
+                            temp_secret_file_dir: "./meta/secrets/",
                         },
                     ),
                     compute_opts: Some(
@@ -375,6 +491,7 @@ mod test {
                             async_stack_trace: None,
                             heap_profiling_dir: None,
                             connector_rpc_endpoint: None,
+                            temp_secret_file_dir: "./compute/secrets/",
                         },
                     ),
                     frontend_opts: Some(
@@ -391,6 +508,7 @@ mod test {
                             config_path: "src/config/test.toml",
                             metrics_level: None,
                             enable_barrier_read: None,
+                            temp_secret_file_dir: "./frontend/secrets/",
                         },
                     ),
                     compactor_opts: None,
diff --git a/src/common/Cargo.toml b/src/common/Cargo.toml
index 3ae8fb38fcd5d..a117dce645ae6 100644
--- a/src/common/Cargo.toml
+++ b/src/common/Cargo.toml
@@ -86,6 +86,7 @@ risingwave-fields-derive = { path = "./fields-derive" }
 risingwave_common_estimate_size = { workspace = true }
 risingwave_common_metrics = { path = "./metrics" }
 risingwave_common_proc_macro = { workspace = true }
+risingwave_common_secret = { path = "./secret" }
 risingwave_error = { workspace = true }
 risingwave_license = { workspace = true }
 risingwave_pb = { workspace = true }
@@ -164,6 +165,10 @@ harness = false
 name = "bench_hash_key_encoding"
 harness = false
 
+[[bench]]
+name = "bench_column_aware_row_encoding"
+harness = false
+
 [[bench]]
 name = "bench_data_chunk_encoding"
 harness = false
diff --git a/src/common/benches/bench_column_aware_row_encoding.rs b/src/common/benches/bench_column_aware_row_encoding.rs
new file mode 100644
index 0000000000000..7a6112800c9b0
--- /dev/null
+++ b/src/common/benches/bench_column_aware_row_encoding.rs
@@ -0,0 +1,136 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::sync::Arc;
+
+use criterion::{black_box, criterion_group, criterion_main, Criterion};
+use rand::{Rng, SeedableRng};
+use risingwave_common::catalog::ColumnId;
+use risingwave_common::row::OwnedRow;
+use risingwave_common::types::{DataType, Date, ScalarImpl};
+use risingwave_common::util::value_encoding::column_aware_row_encoding::*;
+use risingwave_common::util::value_encoding::*;
+
+fn bench_column_aware_encoding_16_columns(c: &mut Criterion) {
+    let mut rng = rand::rngs::StdRng::seed_from_u64(42);
+
+    // The schema is inspired by the TPC-H lineitem table
+    let data_types = Arc::new([
+        DataType::Int64,
+        DataType::Int64,
+        DataType::Int64,
+        DataType::Int32,
+        DataType::Decimal,
+        DataType::Decimal,
+        DataType::Decimal,
+        DataType::Decimal,
+        DataType::Varchar,
+        DataType::Varchar,
+        DataType::Date,
+        DataType::Date,
+        DataType::Date,
+        DataType::Varchar,
+        DataType::Varchar,
+        DataType::Varchar,
+    ]);
+    let row = OwnedRow::new(vec![
+        Some(ScalarImpl::Int64(rng.gen())),
+        Some(ScalarImpl::Int64(rng.gen())),
+        Some(ScalarImpl::Int64(rng.gen())),
+        Some(ScalarImpl::Int32(rng.gen())),
+        Some(ScalarImpl::Decimal("1.0".parse().unwrap())),
+        Some(ScalarImpl::Decimal("114.514".parse().unwrap())),
+        None,
+        Some(ScalarImpl::Decimal("0.08".parse().unwrap())),
+        Some(ScalarImpl::Utf8("A".into())),
+        Some(ScalarImpl::Utf8("B".into())),
+        Some(ScalarImpl::Date(Date::from_ymd_uncheck(2024, 7, 1))),
+        Some(ScalarImpl::Date(Date::from_ymd_uncheck(2024, 7, 2))),
+        Some(ScalarImpl::Date(Date::from_ymd_uncheck(2024, 7, 3))),
+        Some(ScalarImpl::Utf8("D".into())),
+        None,
+        Some(ScalarImpl::Utf8("No comments".into())),
+    ]);
+
+    let column_ids = (1..=data_types.len())
+        .map(|i| ColumnId::from(i as i32))
+        .collect::<Vec<_>>();
+
+    c.bench_function("column_aware_row_encoding_16_columns_encode", |b| {
+        let serializer = Serializer::new(&column_ids[..]);
+        b.iter(|| {
+            black_box(serializer.serialize(&row));
+        });
+    });
+
+    let serializer = Serializer::new(&column_ids[..]);
+    let encoded = serializer.serialize(&row);
+
+    c.bench_function("column_aware_row_encoding_16_columns_decode", |b| {
+        let deserializer =
+            Deserializer::new(&column_ids[..], data_types.clone(), std::iter::empty());
+        b.iter(|| {
+            let result = deserializer.deserialize(&encoded).unwrap();
+            black_box(result);
+        });
+    });
+}
+
+fn bench_column_aware_encoding_4_columns(c: &mut Criterion) {
+    let mut rng = rand::rngs::StdRng::seed_from_u64(42);
+
+    // The schema is inspired by the TPC-H nation table
+    let data_types = Arc::new([
+        DataType::Int32,
+        DataType::Varchar,
+        DataType::Int32,
+        DataType::Varchar,
+    ]);
+    let row = OwnedRow::new(vec![
+        Some(ScalarImpl::Int32(rng.gen())),
+        Some(ScalarImpl::Utf8("United States".into())),
+        Some(ScalarImpl::Int32(rng.gen())),
+        Some(ScalarImpl::Utf8("No comments".into())),
+    ]);
+
+    let column_ids = (1..=data_types.len())
+        .map(|i| ColumnId::from(i as i32))
+        .collect::<Vec<_>>();
+
+    c.bench_function("column_aware_row_encoding_4_columns_encode", |b| {
+        let serializer = Serializer::new(&column_ids[..]);
+        b.iter(|| {
+            black_box(serializer.serialize(&row));
+        });
+    });
+
+    let serializer = Serializer::new(&column_ids[..]);
+    let encoded = serializer.serialize(&row);
+
+    c.bench_function("column_aware_row_encoding_4_columns_decode", |b| {
+        let deserializer =
+            Deserializer::new(&column_ids[..], data_types.clone(), std::iter::empty());
+        b.iter(|| {
+            let result = deserializer.deserialize(&encoded).unwrap();
+            black_box(result);
+        });
+    });
+}
+
+criterion_group!(
+    benches,
+    bench_column_aware_encoding_16_columns,
+    bench_column_aware_encoding_4_columns,
+);
+criterion_main!(benches);
diff --git a/src/common/benches/bitmap.rs b/src/common/benches/bitmap.rs
index 97a1e2b6d837b..e064613d683b4 100644
--- a/src/common/benches/bitmap.rs
+++ b/src/common/benches/bitmap.rs
@@ -14,7 +14,7 @@
 
 use criterion::{black_box, criterion_group, criterion_main, BatchSize, Criterion};
 use itertools::Itertools;
-use risingwave_common::buffer::{Bitmap, BitmapIter};
+use risingwave_common::bitmap::{Bitmap, BitmapIter};
 
 fn bench_bitmap(c: &mut Criterion) {
     const CHUNK_SIZE: usize = 1024;
diff --git a/src/common/common_service/Cargo.toml b/src/common/common_service/Cargo.toml
index f2fa832eaad55..cb43702f3f9e6 100644
--- a/src/common/common_service/Cargo.toml
+++ b/src/common/common_service/Cargo.toml
@@ -28,7 +28,7 @@ thiserror-ext = { workspace = true }
 tokio = { version = "0.2", package = "madsim-tokio", features = ["rt", "rt-multi-thread", "sync", "macros", "time", "signal"] }
 tonic = { workspace = true }
 tower = { version = "0.4", features = ["util", "load-shed"] }
-tower-http = { version = "0.5", features = ["add-extension"] }
+tower-http = { version = "0.5", features = ["add-extension", "compression-gzip"] }
 tracing = "0.1"
 
 [target.'cfg(not(madsim))'.dependencies]
diff --git a/src/common/common_service/src/lib.rs b/src/common/common_service/src/lib.rs
index c09c84012819b..2cf9a56e076f3 100644
--- a/src/common/common_service/src/lib.rs
+++ b/src/common/common_service/src/lib.rs
@@ -18,9 +18,13 @@
 #![feature(impl_trait_in_assoc_type)]
 #![feature(error_generic_member_access)]
 
-pub mod metrics_manager;
-pub mod observer_manager;
+mod metrics_manager;
+mod observer_manager;
+mod tracing;
 
 pub use metrics_manager::MetricsManager;
-
-pub mod tracing;
+pub use observer_manager::{
+    Channel, NotificationClient, ObserverError, ObserverManager, ObserverState,
+    RpcNotificationClient,
+};
+pub use tracing::TracingExtractLayer;
diff --git a/src/common/common_service/src/metrics_manager.rs b/src/common/common_service/src/metrics_manager.rs
index f6d31c98ef31b..a8151d2c17715 100644
--- a/src/common/common_service/src/metrics_manager.rs
+++ b/src/common/common_service/src/metrics_manager.rs
@@ -23,6 +23,7 @@ use risingwave_common::monitor::GLOBAL_METRICS_REGISTRY;
 use thiserror_ext::AsReport;
 use tokio::net::TcpListener;
 use tower_http::add_extension::AddExtensionLayer;
+use tower_http::compression::CompressionLayer;
 use tracing::{error, info, warn};
 
 pub struct MetricsManager {}
@@ -31,28 +32,30 @@ impl MetricsManager {
     pub fn boot_metrics_service(listen_addr: String) {
         static METRICS_SERVICE_LISTEN_ADDR: OnceLock<String> = OnceLock::new();
         let new_listen_addr = listen_addr.clone();
-        let current_listen_addr =
-            METRICS_SERVICE_LISTEN_ADDR.get_or_init(|| {
-                let listen_addr_clone = listen_addr.clone();
-                #[cfg(not(madsim))] // no need in simulation test
-                tokio::spawn(async move {
-                    info!(
-                        "Prometheus listener for Prometheus is set up on http://{}",
-                        listen_addr
-                    );
+        let current_listen_addr = METRICS_SERVICE_LISTEN_ADDR.get_or_init(|| {
+            let listen_addr_clone = listen_addr.clone();
+            #[cfg(not(madsim))] // no need in simulation test
+            tokio::spawn(async move {
+                info!(
+                    "Prometheus listener for Prometheus is set up on http://{}",
+                    listen_addr
+                );
 
-                    let service = Router::new().fallback(Self::metrics_service).layer(
-                        AddExtensionLayer::new(GLOBAL_METRICS_REGISTRY.deref().clone()),
-                    );
+                let service = Router::new()
+                    .fallback(Self::metrics_service)
+                    .layer(AddExtensionLayer::new(
+                        GLOBAL_METRICS_REGISTRY.deref().clone(),
+                    ))
+                    .layer(CompressionLayer::new());
 
-                    let serve_future =
-                        axum::serve(TcpListener::bind(&listen_addr).await.unwrap(), service);
-                    if let Err(err) = serve_future.await {
-                        error!(error = %err.as_report(), "metrics service exited with error");
-                    }
-                });
-                listen_addr_clone
+                let serve_future =
+                    axum::serve(TcpListener::bind(&listen_addr).await.unwrap(), service);
+                if let Err(err) = serve_future.await {
+                    error!(error = %err.as_report(), "metrics service exited with error");
+                }
             });
+            listen_addr_clone
+        });
         if new_listen_addr != *current_listen_addr {
             warn!(
                 "unable to listen port {} for metrics service. Currently listening on {}",
diff --git a/src/common/common_service/src/observer_manager.rs b/src/common/common_service/src/observer_manager.rs
index e760a0e16866c..da61118f1ad49 100644
--- a/src/common/common_service/src/observer_manager.rs
+++ b/src/common/common_service/src/observer_manager.rs
@@ -22,42 +22,6 @@ use thiserror_ext::AsReport;
 use tokio::task::JoinHandle;
 use tonic::{Status, Streaming};
 
-pub trait SubscribeTypeEnum {
-    fn subscribe_type() -> SubscribeType;
-}
-
-pub struct SubscribeFrontend {}
-
-impl SubscribeTypeEnum for SubscribeFrontend {
-    fn subscribe_type() -> SubscribeType {
-        SubscribeType::Frontend
-    }
-}
-
-pub struct SubscribeHummock {}
-
-impl SubscribeTypeEnum for SubscribeHummock {
-    fn subscribe_type() -> SubscribeType {
-        SubscribeType::Hummock
-    }
-}
-
-pub struct SubscribeCompactor {}
-
-impl SubscribeTypeEnum for SubscribeCompactor {
-    fn subscribe_type() -> SubscribeType {
-        SubscribeType::Compactor
-    }
-}
-
-pub struct SubscribeCompute {}
-
-impl SubscribeTypeEnum for SubscribeCompute {
-    fn subscribe_type() -> SubscribeType {
-        SubscribeType::Compute
-    }
-}
-
 /// `ObserverManager` is used to update data based on notification from meta.
 /// Call `start` to spawn a new asynchronous task
 /// We can write the notification logic by implementing `ObserverNodeImpl`.
@@ -68,7 +32,7 @@ pub struct ObserverManager<T: NotificationClient, S: ObserverState> {
 }
 
 pub trait ObserverState: Send + 'static {
-    type SubscribeType: SubscribeTypeEnum;
+    fn subscribe_type() -> SubscribeType;
     /// modify data after receiving notification from meta
     fn handle_notification(&mut self, resp: SubscribeResponse);
 
@@ -109,10 +73,7 @@ where
     S: ObserverState,
 {
     pub async fn new(client: T, observer_states: S) -> Self {
-        let rx = client
-            .subscribe(S::SubscribeType::subscribe_type())
-            .await
-            .unwrap();
+        let rx = client.subscribe(S::subscribe_type()).await.unwrap();
         Self {
             rx,
             client,
@@ -196,7 +157,7 @@ where
                 match self.rx.message().await {
                     Ok(resp) => {
                         if resp.is_none() {
-                            tracing::error!("Stream of notification terminated.");
+                            tracing::warn!("Stream of notification terminated.");
                             self.re_subscribe().await;
                             continue;
                         }
@@ -214,11 +175,7 @@ where
     /// `re_subscribe` is used to re-subscribe to the meta's notification.
     async fn re_subscribe(&mut self) {
         loop {
-            match self
-                .client
-                .subscribe(S::SubscribeType::subscribe_type())
-                .await
-            {
+            match self.client.subscribe(S::subscribe_type()).await {
                 Ok(rx) => {
                     tracing::debug!("re-subscribe success");
                     self.rx = rx;
diff --git a/src/common/estimate_size/src/collections/btreemap.rs b/src/common/estimate_size/src/collections/btreemap.rs
index 84ecd687d8df1..f48a78715f692 100644
--- a/src/common/estimate_size/src/collections/btreemap.rs
+++ b/src/common/estimate_size/src/collections/btreemap.rs
@@ -12,11 +12,13 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use core::fmt;
 use std::collections::BTreeMap;
 use std::ops::{Bound, RangeInclusive};
 
 use crate::{EstimateSize, KvSize};
 
+#[derive(Clone)]
 pub struct EstimatedBTreeMap<K, V> {
     inner: BTreeMap<K, V>,
     heap_size: KvSize,
@@ -41,40 +43,81 @@ impl<K, V> EstimatedBTreeMap<K, V> {
     pub fn is_empty(&self) -> bool {
         self.inner.is_empty()
     }
-}
 
-impl<K, V> Default for EstimatedBTreeMap<K, V> {
-    fn default() -> Self {
-        Self::new()
+    pub fn iter(&self) -> impl Iterator<Item = (&K, &V)> {
+        self.inner.iter()
+    }
+
+    pub fn range<R>(&self, range: R) -> std::collections::btree_map::Range<'_, K, V>
+    where
+        K: Ord,
+        R: std::ops::RangeBounds<K>,
+    {
+        self.inner.range(range)
+    }
+
+    pub fn values(&self) -> impl Iterator<Item = &V> {
+        self.inner.values()
     }
 }
 
 impl<K, V> EstimatedBTreeMap<K, V>
 where
-    K: EstimateSize + Ord,
-    V: EstimateSize,
+    K: Ord,
 {
     pub fn first_key_value(&self) -> Option<(&K, &V)> {
         self.inner.first_key_value()
     }
 
+    pub fn first_key(&self) -> Option<&K> {
+        self.first_key_value().map(|(k, _)| k)
+    }
+
+    pub fn first_value(&self) -> Option<&V> {
+        self.first_key_value().map(|(_, v)| v)
+    }
+
     pub fn last_key_value(&self) -> Option<(&K, &V)> {
         self.inner.last_key_value()
     }
 
-    pub fn insert(&mut self, key: K, row: V) {
+    pub fn last_key(&self) -> Option<&K> {
+        self.last_key_value().map(|(k, _)| k)
+    }
+
+    pub fn last_value(&self) -> Option<&V> {
+        self.last_key_value().map(|(_, v)| v)
+    }
+}
+
+impl<K, V> Default for EstimatedBTreeMap<K, V> {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl<K, V> EstimatedBTreeMap<K, V>
+where
+    K: EstimateSize + Ord,
+    V: EstimateSize,
+{
+    pub fn insert(&mut self, key: K, value: V) -> Option<V> {
         let key_size = self.heap_size.add_val(&key);
-        self.heap_size.add_val(&row);
-        if let Some(old_row) = self.inner.insert(key, row) {
+        self.heap_size.add_val(&value);
+        let old_value = self.inner.insert(key, value);
+        if let Some(old_value) = &old_value {
             self.heap_size.sub_size(key_size);
-            self.heap_size.sub_val(&old_row);
+            self.heap_size.sub_val(old_value);
         }
+        old_value
     }
 
-    pub fn remove(&mut self, key: &K) {
-        if let Some(row) = self.inner.remove(key) {
-            self.heap_size.sub(key, &row);
+    pub fn remove(&mut self, key: &K) -> Option<V> {
+        let old_value = self.inner.remove(key);
+        if let Some(old_value) = &old_value {
+            self.heap_size.sub(key, old_value);
         }
+        old_value
     }
 
     pub fn clear(&mut self) {
@@ -82,6 +125,25 @@ where
         self.heap_size.set(0);
     }
 
+    pub fn pop_first(&mut self) -> Option<(K, V)> {
+        let (key, value) = self.inner.pop_first()?;
+        self.heap_size.sub(&key, &value);
+        Some((key, value))
+    }
+
+    pub fn pop_last(&mut self) -> Option<(K, V)> {
+        let (key, value) = self.inner.pop_last()?;
+        self.heap_size.sub(&key, &value);
+        Some((key, value))
+    }
+
+    pub fn last_entry(&mut self) -> Option<OccupiedEntry<'_, K, V>> {
+        self.inner.last_entry().map(|inner| OccupiedEntry {
+            inner,
+            heap_size: &mut self.heap_size,
+        })
+    }
+
     /// Retain the given range of entries in the map, removing others.
     pub fn retain_range(&mut self, range: RangeInclusive<&K>) -> (BTreeMap<K, V>, BTreeMap<K, V>)
     where
@@ -114,6 +176,27 @@ where
 
         (left, right)
     }
+
+    pub fn extract_if<'a, F>(
+        &'a mut self,
+        mut pred: F,
+    ) -> ExtractIf<'a, K, V, impl FnMut(&K, &mut V) -> bool>
+    where
+        F: 'a + FnMut(&K, &V) -> bool,
+    {
+        let pred_immut = move |key: &K, value: &mut V| pred(key, value);
+        ExtractIf {
+            inner: self.inner.extract_if(pred_immut),
+            heap_size: &mut self.heap_size,
+        }
+    }
+
+    pub fn retain<F>(&mut self, mut f: F)
+    where
+        F: FnMut(&K, &V) -> bool,
+    {
+        self.extract_if(|k, v| !f(k, v)).for_each(drop);
+    }
 }
 
 impl<K, V> EstimateSize for EstimatedBTreeMap<K, V>
@@ -126,6 +209,64 @@ where
     }
 }
 
+impl<K, V> fmt::Debug for EstimatedBTreeMap<K, V>
+where
+    K: fmt::Debug,
+    V: fmt::Debug,
+{
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        self.inner.fmt(f)
+    }
+}
+
+pub struct OccupiedEntry<'a, K, V> {
+    inner: std::collections::btree_map::OccupiedEntry<'a, K, V>,
+    heap_size: &'a mut KvSize,
+}
+
+impl<'a, K, V> OccupiedEntry<'a, K, V>
+where
+    K: EstimateSize + Ord,
+    V: EstimateSize,
+{
+    pub fn key(&self) -> &K {
+        self.inner.key()
+    }
+
+    pub fn remove_entry(self) -> (K, V) {
+        let (key, value) = self.inner.remove_entry();
+        self.heap_size.sub(&key, &value);
+        (key, value)
+    }
+}
+
+pub struct ExtractIf<'a, K, V, F>
+where
+    F: FnMut(&K, &mut V) -> bool,
+{
+    inner: std::collections::btree_map::ExtractIf<'a, K, V, F>,
+    heap_size: &'a mut KvSize,
+}
+
+impl<'a, K, V, F> Iterator for ExtractIf<'a, K, V, F>
+where
+    K: EstimateSize,
+    V: EstimateSize,
+    F: FnMut(&K, &mut V) -> bool,
+{
+    type Item = (K, V);
+
+    fn next(&mut self) -> Option<Self::Item> {
+        let (key, value) = self.inner.next()?;
+        self.heap_size.sub(&key, &value);
+        Some((key, value))
+    }
+
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        self.inner.size_hint()
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::EstimatedBTreeMap;
diff --git a/src/common/estimate_size/src/lib.rs b/src/common/estimate_size/src/lib.rs
index 32e41fcf87fb2..3629bc5c535f2 100644
--- a/src/common/estimate_size/src/lib.rs
+++ b/src/common/estimate_size/src/lib.rs
@@ -14,6 +14,7 @@
 
 #![feature(allocator_api)]
 #![feature(btree_cursors)]
+#![feature(btree_extract_if)]
 
 pub mod collections;
 
diff --git a/src/common/metrics/src/lib.rs b/src/common/metrics/src/lib.rs
index fa0250f4f0c7b..35e8b11265843 100644
--- a/src/common/metrics/src/lib.rs
+++ b/src/common/metrics/src/lib.rs
@@ -12,7 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#![feature(lazy_cell)]
 #![feature(type_alias_impl_trait)]
 #![feature(impl_trait_in_assoc_type)]
 use std::ops::Deref;
diff --git a/src/common/metrics/src/monitor/connection.rs b/src/common/metrics/src/monitor/connection.rs
index 295fb6399ba4b..e5774a3f16d7d 100644
--- a/src/common/metrics/src/monitor/connection.rs
+++ b/src/common/metrics/src/monitor/connection.rs
@@ -587,28 +587,6 @@ impl<L> tonic::transport::server::Router<L> {
     }
 }
 
-#[cfg(not(madsim))]
-pub fn monitored_tcp_incoming(
-    listen_addr: std::net::SocketAddr,
-    connection_type: impl Into<String>,
-    config: TcpConfig,
-) -> Result<
-    MonitoredConnection<tonic::transport::server::TcpIncoming, MonitorNewConnectionImpl>,
-    Box<dyn std::error::Error + Send + Sync>,
-> {
-    let incoming = tonic::transport::server::TcpIncoming::new(
-        listen_addr,
-        config.tcp_nodelay,
-        config.keepalive_duration,
-    )?;
-    Ok(MonitoredConnection::new(
-        incoming,
-        MonitorNewConnectionImpl {
-            connection_type: connection_type.into(),
-        },
-    ))
-}
-
 #[derive(Clone)]
 pub struct MonitorNewConnectionImpl {
     connection_type: String,
diff --git a/src/common/metrics/src/monitor/mod.rs b/src/common/metrics/src/monitor/mod.rs
index 10b5c966e636a..316cac9ea907c 100644
--- a/src/common/metrics/src/monitor/mod.rs
+++ b/src/common/metrics/src/monitor/mod.rs
@@ -12,20 +12,16 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-pub mod connection;
-pub mod my_stats;
-pub mod process;
-pub mod rwlock;
+pub use connection::{monitor_connector, EndpointExt, RouterExt, TcpConfig};
+pub use rwlock::MonitoredRwLock;
 
-use std::sync::LazyLock;
+mod connection;
+mod process;
+mod rwlock;
 
-use prometheus::core::{
-    AtomicI64, AtomicU64, Collector, GenericCounter, GenericCounterVec, GenericGauge, Metric,
-};
-use prometheus::{Histogram, HistogramVec, Registry};
+use std::sync::LazyLock;
 
-use crate::monitor::my_stats::MyHistogram;
-use crate::monitor::process::monitor_process;
+use prometheus::Registry;
 
 #[cfg(target_os = "linux")]
 static PAGESIZE: std::sync::LazyLock<i64> =
@@ -35,59 +31,8 @@ static PAGESIZE: std::sync::LazyLock<i64> =
 pub static CLOCK_TICK: std::sync::LazyLock<u64> =
     std::sync::LazyLock::new(|| unsafe { libc::sysconf(libc::_SC_CLK_TCK) as u64 });
 
-/// Define extension method `print` used in `print_statistics`.
-pub trait Print {
-    fn print(&self);
-}
-
-impl Print for GenericCounter<AtomicU64> {
-    fn print(&self) {
-        let desc = &self.desc()[0].fq_name;
-        let counter = self.metric().get_counter().get_value() as u64;
-        println!("{desc} COUNT : {counter}");
-    }
-}
-
-impl Print for GenericGauge<AtomicI64> {
-    fn print(&self) {
-        let desc = &self.desc()[0].fq_name;
-        let counter = self.get();
-        println!("{desc} COUNT : {counter}");
-    }
-}
-
-impl Print for Histogram {
-    fn print(&self) {
-        let desc = &self.desc()[0].fq_name;
-
-        let histogram = MyHistogram::from_prom_hist(self.metric().get_histogram());
-        let p50 = histogram.get_percentile(50.0);
-        let p95 = histogram.get_percentile(95.0);
-        let p99 = histogram.get_percentile(99.0);
-        let p100 = histogram.get_percentile(100.0);
-
-        let sample_count = self.get_sample_count();
-        let sample_sum = self.get_sample_sum();
-        println!("{desc} P50 : {p50} P95 : {p95} P99 : {p99} P100 : {p100} COUNT : {sample_count} SUM : {sample_sum}");
-    }
-}
-
-impl Print for HistogramVec {
-    fn print(&self) {
-        let desc = &self.desc()[0].fq_name;
-        println!("{desc} {:?}", self);
-    }
-}
-
-impl Print for GenericCounterVec<AtomicU64> {
-    fn print(&self) {
-        let desc = &self.desc()[0].fq_name;
-        println!("{desc} {:?}", self);
-    }
-}
-
 pub static GLOBAL_METRICS_REGISTRY: LazyLock<Registry> = LazyLock::new(|| {
     let registry = Registry::new();
-    monitor_process(&registry);
+    process::monitor_process(&registry);
     registry
 });
diff --git a/src/common/metrics/src/monitor/my_stats.rs b/src/common/metrics/src/monitor/my_stats.rs
deleted file mode 100644
index 52c71167f2f97..0000000000000
--- a/src/common/metrics/src/monitor/my_stats.rs
+++ /dev/null
@@ -1,221 +0,0 @@
-// Copyright 2024 RisingWave Labs
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::fmt::{Display, Formatter};
-
-use itertools::Itertools;
-use prometheus::proto::Histogram;
-use rw_iter_util::ZipEqFast;
-
-#[derive(Clone, Default, Debug)]
-pub struct MyHistogram {
-    pub upper_bound_list: Vec<f64>,
-    pub count_list: Vec<u64>,
-    pub total_count: u64,
-    pub total_sum: f64,
-}
-
-impl MyHistogram {
-    pub fn from_prom_hist(histogram: &Histogram) -> MyHistogram {
-        let mut upper_bound_list = Vec::new();
-        let mut count_list = Vec::new();
-
-        let total_count = histogram.get_sample_count();
-        let total_sum = histogram.get_sample_sum();
-
-        let buckets = histogram.get_bucket();
-        for bucket in buckets {
-            let upper_bound = bucket.get_upper_bound();
-            let count = bucket.get_cumulative_count();
-            upper_bound_list.push(upper_bound);
-            count_list.push(count);
-        }
-
-        MyHistogram {
-            upper_bound_list,
-            count_list,
-            total_count,
-            total_sum,
-        }
-    }
-
-    pub fn from_diff(prev: &MyHistogram, cur: &MyHistogram) -> MyHistogram {
-        MyHistogram {
-            upper_bound_list: cur.upper_bound_list.clone(),
-            count_list: match prev.count_list.is_empty() {
-                true => cur.count_list.clone(),
-                false => prev
-                    .count_list
-                    .iter()
-                    .zip_eq_fast(cur.count_list.iter())
-                    .map(|(&pb, &cb)| cb - pb)
-                    .collect_vec(),
-            },
-            total_sum: cur.total_sum - prev.total_sum,
-            total_count: cur.total_count - prev.total_count,
-        }
-    }
-
-    pub fn get_percentile(&self, p: f64) -> f64 {
-        let sample_count = self.total_count;
-
-        // empty bucket may appear
-        if sample_count == 0 {
-            return 0.0;
-        }
-        let threshold = (sample_count as f64 * (p / 100.0_f64)).ceil() as u64;
-        let mut last_upper_bound = 0.0;
-        let mut last_count = 0;
-        for (&upper_bound, &count) in self
-            .upper_bound_list
-            .iter()
-            .zip_eq_fast(self.count_list.iter())
-        {
-            if count >= threshold {
-                // assume scale linearly within this bucket,
-                // return a value between last_upper_bound and upper_bound
-                let right_left_diff = upper_bound - last_upper_bound;
-                return last_upper_bound
-                    + right_left_diff * (threshold - last_count) as f64
-                        / (count - last_count) as f64;
-            }
-            last_upper_bound = upper_bound;
-            last_count = count;
-        }
-
-        0.0
-    }
-}
-
-impl Display for MyHistogram {
-    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-        // calculate latencies statistics
-        let mean = self.total_sum / self.total_count as f64;
-        let p50 = self.get_percentile(50.0);
-        let p90 = self.get_percentile(90.0);
-        let p99 = self.get_percentile(99.0);
-        let p100 = self.get_percentile(100.0);
-
-        write!(
-            f,
-            "latency:
-        mean: {},
-        p50: {},
-        p90: {},
-        p99: {},
-        p100: {};",
-            mean, p50, p90, p99, p100
-        )
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use prometheus::core::Metric;
-    use prometheus::{histogram_opts, register_histogram_with_registry, Registry};
-
-    use super::*;
-
-    #[test]
-    fn test_proc_histogram_basic() {
-        fn new_simple_histogram(upper_bound: u64) -> MyHistogram {
-            let registry = Registry::new();
-            let buckets = (1..=upper_bound).map(|x| x as f64).collect::<Vec<f64>>();
-            let opts = histogram_opts!("test_histogram", "test_histogram", buckets);
-
-            let histogram = register_histogram_with_registry!(opts, registry).unwrap();
-
-            for value in 1..=upper_bound {
-                histogram.observe(value as f64);
-            }
-
-            MyHistogram::from_prom_hist(histogram.metric().get_histogram())
-        }
-
-        let histogram = new_simple_histogram(999);
-        assert_eq!(histogram.get_percentile(50.0) as u64, 500);
-        assert_eq!(histogram.get_percentile(90.0) as u64, 900);
-        assert_eq!(histogram.get_percentile(99.0) as u64, 990);
-        assert_eq!(histogram.get_percentile(99.9) as u64, 999);
-        assert_eq!(histogram.get_percentile(100.0) as u64, 999);
-
-        let histogram = new_simple_histogram(1000);
-        assert_eq!(histogram.get_percentile(50.0) as u64, 500);
-        assert_eq!(histogram.get_percentile(90.0) as u64, 900);
-        assert_eq!(histogram.get_percentile(99.0) as u64, 990);
-        assert_eq!(histogram.get_percentile(99.9) as u64, 1000);
-        assert_eq!(histogram.get_percentile(100.0) as u64, 1000);
-
-        let histogram = new_simple_histogram(9999);
-        assert_eq!(histogram.get_percentile(50.0) as u64, 5000);
-        assert_eq!(histogram.get_percentile(90.0) as u64, 9000);
-        assert_eq!(histogram.get_percentile(99.0) as u64, 9900);
-        assert_eq!(histogram.get_percentile(99.9) as u64, 9990);
-        assert_eq!(histogram.get_percentile(100.0) as u64, 9999);
-    }
-
-    #[test]
-    fn test_proc_histogram_uneven_distributed() {
-        let registry = Registry::new();
-        let buckets = vec![
-            0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0,
-        ];
-        let opts = histogram_opts!("test_histogram", "test_histogram", buckets);
-        let histogram = register_histogram_with_registry!(opts, registry).unwrap();
-
-        let mut i = 0.005;
-        while i < 10.0 {
-            histogram.observe(i);
-            i += 0.005;
-        }
-
-        let histogram = MyHistogram::from_prom_hist(histogram.metric().get_histogram());
-        assert_eq!(histogram.get_percentile(50.0), 5.0);
-        assert_eq!(histogram.get_percentile(90.0), 9.004004004004004);
-        assert_eq!(histogram.get_percentile(99.0), 9.904904904904905);
-        assert_eq!(histogram.get_percentile(99.9), 9.994994994994995);
-        assert_eq!(histogram.get_percentile(100.0), 10.0);
-    }
-
-    #[test]
-    fn test_proc_histogram_realistic() {
-        let registry = Registry::new();
-        let buckets = vec![
-            0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0,
-        ];
-        let opts = histogram_opts!("test_histogram", "test_histogram", buckets);
-        let histogram = register_histogram_with_registry!(opts, registry).unwrap();
-
-        histogram.observe(0.0012);
-        histogram.observe(0.0013);
-        histogram.observe(0.003);
-
-        histogram.observe(0.0132);
-        histogram.observe(0.0143);
-        histogram.observe(0.0146);
-        histogram.observe(0.0249);
-
-        histogram.observe(0.99);
-
-        histogram.observe(6.11);
-        histogram.observe(7.833);
-
-        let histogram = MyHistogram::from_prom_hist(histogram.metric().get_histogram());
-        assert_eq!(histogram.get_percentile(50.0), 0.0175);
-        assert_eq!(histogram.get_percentile(90.0), 7.5);
-        assert_eq!(histogram.get_percentile(99.0), 10.00);
-        assert_eq!(histogram.get_percentile(99.9), 10.00);
-        assert_eq!(histogram.get_percentile(100.0), 10.00);
-    }
-}
diff --git a/src/common/secret/Cargo.toml b/src/common/secret/Cargo.toml
new file mode 100644
index 0000000000000..4b698a737cb54
--- /dev/null
+++ b/src/common/secret/Cargo.toml
@@ -0,0 +1,29 @@
+[package]
+name = "risingwave_common_secret"
+version = { workspace = true }
+edition = { workspace = true }
+homepage = { workspace = true }
+keywords = { workspace = true }
+license = { workspace = true }
+repository = { workspace = true }
+
+[package.metadata.cargo-machete]
+ignored = ["workspace-hack"]
+
+[package.metadata.cargo-udeps.ignore]
+normal = ["workspace-hack"]
+
+[dependencies]
+aes-gcm = "0.10"
+anyhow = "1"
+bincode = "1"
+parking_lot = { workspace = true }
+prost = { workspace = true }
+risingwave_pb = { workspace = true }
+serde = { version = "1" }
+thiserror = "1"
+thiserror-ext = { workspace = true }
+tracing = "0.1"
+
+[lints]
+workspace = true
diff --git a/src/common/secret/src/encryption.rs b/src/common/secret/src/encryption.rs
new file mode 100644
index 0000000000000..a6c0253fb1f9c
--- /dev/null
+++ b/src/common/secret/src/encryption.rs
@@ -0,0 +1,80 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use aes_gcm::aead::generic_array::GenericArray;
+use aes_gcm::aead::{Aead, AeadCore, KeyInit, OsRng};
+use aes_gcm::Aes128Gcm;
+use serde::{Deserialize, Serialize};
+
+use super::{SecretError, SecretResult};
+
+#[derive(Deserialize, Serialize)]
+pub struct SecretEncryption {
+    nonce: [u8; 12],
+    ciphertext: Vec<u8>,
+}
+
+impl SecretEncryption {
+    pub fn encrypt(key: &[u8], plaintext: &[u8]) -> SecretResult<Self> {
+        let encrypt_key = Self::fill_key(key);
+        let nonce_array = Aes128Gcm::generate_nonce(&mut OsRng);
+        let cipher = Aes128Gcm::new(encrypt_key.as_slice().into());
+        let ciphertext = cipher
+            .encrypt(&nonce_array, plaintext)
+            .map_err(|_| SecretError::AesError)?;
+        Ok(Self {
+            nonce: nonce_array.into(),
+            ciphertext,
+        })
+    }
+
+    pub fn decrypt(&self, key: &[u8]) -> SecretResult<Vec<u8>> {
+        let decrypt_key = Self::fill_key(key);
+        let nonce_array = GenericArray::from_slice(&self.nonce);
+        let cipher = Aes128Gcm::new(decrypt_key.as_slice().into());
+        let plaintext = cipher
+            .decrypt(nonce_array, self.ciphertext.as_slice())
+            .map_err(|_| SecretError::AesError)?;
+        Ok(plaintext)
+    }
+
+    fn fill_key(key: &[u8]) -> Vec<u8> {
+        let mut k = key[..(std::cmp::min(key.len(), 16))].to_vec();
+        k.resize_with(16, || 0);
+        k
+    }
+
+    pub fn serialize(&self) -> SecretResult<Vec<u8>> {
+        let res = bincode::serialize(&self)?;
+        Ok(res)
+    }
+
+    pub fn deserialize(data: &[u8]) -> SecretResult<Self> {
+        let res = bincode::deserialize(data)?;
+        Ok(res)
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    #[test]
+    fn test_secret_encryption_decyption() {
+        let key = b"0123456789abcdef";
+        let plaintext = "Hello, world!".as_bytes();
+        let secret = SecretEncryption::encrypt(key, plaintext).unwrap();
+        let decrypted = secret.decrypt(key).unwrap();
+        assert_eq!(plaintext, decrypted.as_slice());
+    }
+}
diff --git a/src/common/secret/src/error.rs b/src/common/secret/src/error.rs
new file mode 100644
index 0000000000000..6db7b7f927e59
--- /dev/null
+++ b/src/common/secret/src/error.rs
@@ -0,0 +1,45 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub use anyhow::anyhow;
+use thiserror::Error;
+use thiserror_ext::Construct;
+
+use super::SecretId;
+
+pub type SecretResult<T> = Result<T, SecretError>;
+
+#[derive(Error, Debug, Construct)]
+pub enum SecretError {
+    #[error("secret not found: {0}")]
+    ItemNotFound(SecretId),
+
+    #[error("decode utf8 error: {0}")]
+    DecodeUtf8Error(#[from] std::string::FromUtf8Error),
+
+    #[error("I/O error: {0}")]
+    IoError(#[from] std::io::Error),
+
+    #[error("unspecified secret ref type: {0}")]
+    UnspecifiedRefType(SecretId),
+
+    #[error("fail to encrypt/decrypt secret")]
+    AesError,
+
+    #[error("ser/de proto message error: {0}")]
+    ProtoError(#[from] bincode::Error),
+
+    #[error(transparent)]
+    Internal(#[from] anyhow::Error),
+}
diff --git a/src/common/secret/src/lib.rs b/src/common/secret/src/lib.rs
new file mode 100644
index 0000000000000..17319a296734d
--- /dev/null
+++ b/src/common/secret/src/lib.rs
@@ -0,0 +1,22 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+type SecretId = u32;
+
+mod secret_manager;
+pub use secret_manager::*;
+mod encryption;
+pub use encryption::*;
+mod error;
+pub use error::*;
diff --git a/src/common/secret/src/secret_manager.rs b/src/common/secret/src/secret_manager.rs
new file mode 100644
index 0000000000000..b6a71a4c3ebe8
--- /dev/null
+++ b/src/common/secret/src/secret_manager.rs
@@ -0,0 +1,187 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::{BTreeMap, HashMap};
+use std::fs::File;
+use std::io::Write;
+use std::path::PathBuf;
+
+use anyhow::{anyhow, Context};
+use parking_lot::RwLock;
+use prost::Message;
+use risingwave_pb::catalog::PbSecret;
+use risingwave_pb::secret::secret_ref::RefAsType;
+use risingwave_pb::secret::PbSecretRef;
+use thiserror_ext::AsReport;
+
+use super::error::{SecretError, SecretResult};
+use super::SecretId;
+
+static INSTANCE: std::sync::OnceLock<LocalSecretManager> = std::sync::OnceLock::new();
+
+#[derive(Debug)]
+pub struct LocalSecretManager {
+    secrets: RwLock<HashMap<SecretId, Vec<u8>>>,
+    /// The local directory used to write secrets into file, so that it can be passed into some libararies
+    secret_file_dir: PathBuf,
+}
+
+impl LocalSecretManager {
+    /// Initialize the secret manager with the given temp file path, cluster id, and encryption key.
+    /// # Panics
+    /// Panics if fail to create the secret file directory.
+    pub fn init(temp_file_dir: String, cluster_id: String, worker_id: u32) {
+        // use `get_or_init` to handle concurrent initialization in single node mode.
+        INSTANCE.get_or_init(|| {
+            let secret_file_dir = PathBuf::from(temp_file_dir)
+                .join(cluster_id)
+                .join(worker_id.to_string());
+            std::fs::remove_dir_all(&secret_file_dir).ok();
+
+            // This will cause file creation conflict in simulation tests.
+            // Should skip testing secret files in simulation tests.
+            #[cfg(not(madsim))]
+            std::fs::create_dir_all(&secret_file_dir).unwrap();
+
+            Self {
+                secrets: RwLock::new(HashMap::new()),
+                secret_file_dir,
+            }
+        });
+    }
+
+    /// Get the global secret manager instance.
+    /// # Panics
+    /// Panics if the secret manager is not initialized.
+    pub fn global() -> &'static LocalSecretManager {
+        // Initialize the secret manager for unit tests.
+        #[cfg(debug_assertions)]
+        LocalSecretManager::init("./tmp".to_string(), "test_cluster".to_string(), 0);
+
+        INSTANCE.get().unwrap()
+    }
+
+    pub fn add_secret(&self, secret_id: SecretId, secret: Vec<u8>) {
+        let mut secret_guard = self.secrets.write();
+        secret_guard.insert(secret_id, secret);
+    }
+
+    pub fn init_secrets(&self, secrets: Vec<PbSecret>) {
+        let mut secret_guard = self.secrets.write();
+        // Reset the secrets
+        secret_guard.clear();
+        // Error should only occurs when running simulation tests when we have multiple nodes
+        // in 1 process and can fail .
+        std::fs::remove_dir_all(&self.secret_file_dir)
+            .inspect_err(|e| {
+                tracing::error!(
+            error = %e.as_report(),
+            path = %self.secret_file_dir.to_string_lossy(),
+            "Failed to remove secret directory")
+            })
+            .ok();
+
+        #[cfg(not(madsim))]
+        std::fs::create_dir_all(&self.secret_file_dir).unwrap();
+
+        for secret in secrets {
+            secret_guard.insert(secret.id, secret.value);
+        }
+    }
+
+    pub fn get_secret(&self, secret_id: SecretId) -> Option<Vec<u8>> {
+        let secret_guard = self.secrets.read();
+        secret_guard.get(&secret_id).cloned()
+    }
+
+    pub fn remove_secret(&self, secret_id: SecretId) {
+        let mut secret_guard = self.secrets.write();
+        secret_guard.remove(&secret_id);
+        self.remove_secret_file_if_exist(&secret_id);
+    }
+
+    pub fn fill_secrets(
+        &self,
+        mut options: BTreeMap<String, String>,
+        secret_refs: BTreeMap<String, PbSecretRef>,
+    ) -> SecretResult<BTreeMap<String, String>> {
+        let secret_guard = self.secrets.read();
+        for (option_key, secret_ref) in secret_refs {
+            let secret_id = secret_ref.secret_id;
+            let pb_secret_bytes = secret_guard
+                .get(&secret_id)
+                .ok_or(SecretError::ItemNotFound(secret_id))?;
+            let secret_value_bytes = Self::get_secret_value(pb_secret_bytes)?;
+            match secret_ref.ref_as() {
+                RefAsType::Text => {
+                    // We converted the secret string from sql to bytes using `as_bytes` in frontend.
+                    // So use `from_utf8` here to convert it back to string.
+                    options.insert(option_key, String::from_utf8(secret_value_bytes.clone())?);
+                }
+                RefAsType::File => {
+                    let path_str =
+                        self.get_or_init_secret_file(secret_id, secret_value_bytes.clone())?;
+                    options.insert(option_key, path_str);
+                }
+                RefAsType::Unspecified => {
+                    return Err(SecretError::UnspecifiedRefType(secret_id));
+                }
+            }
+        }
+        Ok(options)
+    }
+
+    /// Get the secret file for the given secret id and return the path string. If the file does not exist, create it.
+    /// WARNING: This method should be called only when the secret manager is locked.
+    fn get_or_init_secret_file(
+        &self,
+        secret_id: SecretId,
+        secret_bytes: Vec<u8>,
+    ) -> SecretResult<String> {
+        let path = self.secret_file_dir.join(secret_id.to_string());
+        if !path.exists() {
+            let mut file = File::create(&path)?;
+            file.write_all(&secret_bytes)?;
+            file.sync_all()?;
+        }
+        Ok(path.to_string_lossy().to_string())
+    }
+
+    /// WARNING: This method should be called only when the secret manager is locked.
+    fn remove_secret_file_if_exist(&self, secret_id: &SecretId) {
+        let path = self.secret_file_dir.join(secret_id.to_string());
+        if path.exists() {
+            std::fs::remove_file(&path)
+                .inspect_err(|e| {
+                    tracing::error!(
+                error = %e.as_report(),
+                path = %path.to_string_lossy(),
+                "Failed to remove secret file")
+                })
+                .ok();
+        }
+    }
+
+    fn get_secret_value(pb_secret_bytes: &[u8]) -> SecretResult<Vec<u8>> {
+        let pb_secret = risingwave_pb::secret::Secret::decode(pb_secret_bytes)
+            .context("failed to decode secret")?;
+        let secret_value = match pb_secret.get_secret_backend().unwrap() {
+            risingwave_pb::secret::secret::SecretBackend::Meta(backend) => backend.value.clone(),
+            risingwave_pb::secret::secret::SecretBackend::HashicorpVault(_) => {
+                return Err(anyhow!("hashicorp_vault backend is not implemented yet").into())
+            }
+        };
+        Ok(secret_value)
+    }
+}
diff --git a/src/common/src/array/arrow/arrow_deltalake.rs b/src/common/src/array/arrow/arrow_deltalake.rs
index 7dd6478ba50f7..7338532e082d6 100644
--- a/src/common/src/array/arrow/arrow_deltalake.rs
+++ b/src/common/src/array/arrow/arrow_deltalake.rs
@@ -107,7 +107,7 @@ mod test {
 
     use crate::array::arrow::arrow_deltalake::DeltaLakeConvert;
     use crate::array::{ArrayImpl, Decimal, DecimalArray, ListArray, ListValue};
-    use crate::buffer::Bitmap;
+    use crate::bitmap::Bitmap;
 
     #[test]
     fn test_decimal_list_chunk() {
diff --git a/src/common/src/array/arrow/arrow_impl.rs b/src/common/src/array/arrow/arrow_impl.rs
index 35057f62f7740..f4ca022ffd7fa 100644
--- a/src/common/src/array/arrow/arrow_impl.rs
+++ b/src/common/src/array/arrow/arrow_impl.rs
@@ -54,6 +54,9 @@ use crate::types::*;
 use crate::util::iter_util::ZipEqFast;
 
 /// Defines how to convert RisingWave arrays to Arrow arrays.
+///
+/// This trait allows for customized conversion logic for different external systems using Arrow.
+/// The default implementation is based on the `From` implemented in this mod.
 pub trait ToArrow {
     /// Converts RisingWave `DataChunk` to Arrow `RecordBatch` with specified schema.
     ///
@@ -519,14 +522,18 @@ pub trait FromArrow {
             Int16 => self.from_int16_array(array.as_any().downcast_ref().unwrap()),
             Int32 => self.from_int32_array(array.as_any().downcast_ref().unwrap()),
             Int64 => self.from_int64_array(array.as_any().downcast_ref().unwrap()),
+            Decimal128(_, _) => self.from_decimal128_array(array.as_any().downcast_ref().unwrap()),
             Decimal256(_, _) => self.from_int256_array(array.as_any().downcast_ref().unwrap()),
             Float32 => self.from_float32_array(array.as_any().downcast_ref().unwrap()),
             Float64 => self.from_float64_array(array.as_any().downcast_ref().unwrap()),
             Date32 => self.from_date32_array(array.as_any().downcast_ref().unwrap()),
             Time64(Microsecond) => self.from_time64us_array(array.as_any().downcast_ref().unwrap()),
-            Timestamp(Microsecond, _) => {
+            Timestamp(Microsecond, None) => {
                 self.from_timestampus_array(array.as_any().downcast_ref().unwrap())
             }
+            Timestamp(Microsecond, Some(_)) => {
+                self.from_timestampus_some_array(array.as_any().downcast_ref().unwrap())
+            }
             Interval(MonthDayNano) => {
                 self.from_interval_array(array.as_any().downcast_ref().unwrap())
             }
@@ -596,6 +603,13 @@ pub trait FromArrow {
         Ok(ArrayImpl::Int256(array.into()))
     }
 
+    fn from_decimal128_array(
+        &self,
+        array: &arrow_array::Decimal128Array,
+    ) -> Result<ArrayImpl, ArrayError> {
+        Ok(ArrayImpl::Decimal(array.try_into()?))
+    }
+
     fn from_float32_array(
         &self,
         array: &arrow_array::Float32Array,
@@ -628,6 +642,13 @@ pub trait FromArrow {
         Ok(ArrayImpl::Timestamp(array.into()))
     }
 
+    fn from_timestampus_some_array(
+        &self,
+        array: &arrow_array::TimestampMicrosecondArray,
+    ) -> Result<ArrayImpl, ArrayError> {
+        Ok(ArrayImpl::Timestamptz(array.into()))
+    }
+
     fn from_interval_array(
         &self,
         array: &arrow_array::IntervalMonthDayNanoArray,
@@ -767,7 +788,7 @@ converts!(IntervalArray, arrow_array::IntervalMonthDayNanoArray, @map);
 converts!(SerialArray, arrow_array::Int64Array, @map);
 
 /// Converts RisingWave value from and into Arrow value.
-pub trait FromIntoArrow {
+trait FromIntoArrow {
     /// The corresponding element type in the Arrow array.
     type ArrowType;
     fn from_arrow(value: Self::ArrowType) -> Self;
@@ -1124,8 +1145,8 @@ mod tests {
     fn date() {
         let array = DateArray::from_iter([
             None,
-            Date::with_days(12345).ok(),
-            Date::with_days(-12345).ok(),
+            Date::with_days_since_ce(12345).ok(),
+            Date::with_days_since_ce(-12345).ok(),
         ]);
         let arrow = arrow_array::Date32Array::from(&array);
         assert_eq!(DateArray::from(&arrow), array);
diff --git a/src/common/src/array/bool_array.rs b/src/common/src/array/bool_array.rs
index ebee50c8e4b72..7ed0e4b703a22 100644
--- a/src/common/src/array/bool_array.rs
+++ b/src/common/src/array/bool_array.rs
@@ -16,7 +16,7 @@ use risingwave_common_estimate_size::EstimateSize;
 use risingwave_pb::data::{ArrayType, PbArray};
 
 use super::{Array, ArrayBuilder, DataType};
-use crate::buffer::{Bitmap, BitmapBuilder};
+use crate::bitmap::{Bitmap, BitmapBuilder};
 
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub struct BoolArray {
diff --git a/src/common/src/array/bytes_array.rs b/src/common/src/array/bytes_array.rs
index 7160c77b0d1c9..0d62c3d26c6e3 100644
--- a/src/common/src/array/bytes_array.rs
+++ b/src/common/src/array/bytes_array.rs
@@ -21,7 +21,7 @@ use risingwave_pb::common::Buffer;
 use risingwave_pb::data::{ArrayType, PbArray};
 
 use super::{Array, ArrayBuilder, DataType};
-use crate::buffer::{Bitmap, BitmapBuilder};
+use crate::bitmap::{Bitmap, BitmapBuilder};
 use crate::util::iter_util::ZipEqDebug;
 
 /// `BytesArray` is a collection of Rust `[u8]`s.
diff --git a/src/common/src/array/chrono_array.rs b/src/common/src/array/chrono_array.rs
index 37a0aa0512a76..cb199fdc01d66 100644
--- a/src/common/src/array/chrono_array.rs
+++ b/src/common/src/array/chrono_array.rs
@@ -35,7 +35,10 @@ mod tests {
 
     #[test]
     fn test_date_builder() {
-        let v = (0..1000).map(Date::with_days).map(|x| x.ok()).collect_vec();
+        let v = (0..1000)
+            .map(Date::with_days_since_ce)
+            .map(|x| x.ok())
+            .collect_vec();
         let mut builder = DateArrayBuilder::new(0);
         for i in &v {
             builder.append(*i);
@@ -48,9 +51,9 @@ mod tests {
     #[test]
     fn test_date_array_to_protobuf() {
         let input = vec![
-            Date::with_days(12345).ok(),
+            Date::with_days_since_ce(12345).ok(),
             None,
-            Date::with_days(67890).ok(),
+            Date::with_days_since_ce(67890).ok(),
         ];
 
         let array = DateArray::from_iter(&input);
diff --git a/src/common/src/array/data_chunk.rs b/src/common/src/array/data_chunk.rs
index 4e08163817e23..a53d9930eba69 100644
--- a/src/common/src/array/data_chunk.rs
+++ b/src/common/src/array/data_chunk.rs
@@ -29,7 +29,7 @@ use risingwave_pb::data::PbDataChunk;
 use super::{Array, ArrayImpl, ArrayRef, ArrayResult, StructArray};
 use crate::array::data_chunk_iter::RowRef;
 use crate::array::ArrayBuilderImpl;
-use crate::buffer::{Bitmap, BitmapBuilder};
+use crate::bitmap::{Bitmap, BitmapBuilder};
 use crate::field_generator::{FieldGeneratorImpl, VarcharProperty};
 use crate::hash::HashCode;
 use crate::row::Row;
diff --git a/src/common/src/array/jsonb_array.rs b/src/common/src/array/jsonb_array.rs
index 4a0df2f55835d..b49b741976714 100644
--- a/src/common/src/array/jsonb_array.rs
+++ b/src/common/src/array/jsonb_array.rs
@@ -16,7 +16,7 @@ use risingwave_common_estimate_size::EstimateSize;
 use risingwave_pb::data::{PbArray, PbArrayType};
 
 use super::{Array, ArrayBuilder, ArrayImpl, ArrayResult};
-use crate::buffer::{Bitmap, BitmapBuilder};
+use crate::bitmap::{Bitmap, BitmapBuilder};
 use crate::types::{DataType, JsonbRef, JsonbVal, Scalar};
 
 #[derive(Debug, Clone, EstimateSize)]
diff --git a/src/common/src/array/list_array.rs b/src/common/src/array/list_array.rs
index 748ab6777fa49..7fc1fdecee6fe 100644
--- a/src/common/src/array/list_array.rs
+++ b/src/common/src/array/list_array.rs
@@ -29,7 +29,7 @@ use super::{
     Array, ArrayBuilder, ArrayBuilderImpl, ArrayImpl, ArrayResult, BoolArray, PrimitiveArray,
     PrimitiveArrayItemType, RowRef, Utf8Array,
 };
-use crate::buffer::{Bitmap, BitmapBuilder};
+use crate::bitmap::{Bitmap, BitmapBuilder};
 use crate::row::Row;
 use crate::types::{
     hash_datum, DataType, Datum, DatumRef, DefaultOrd, Scalar, ScalarImpl, ScalarRefImpl,
diff --git a/src/common/src/array/mod.rs b/src/common/src/array/mod.rs
index 268c518b70c01..89b3b06266786 100644
--- a/src/common/src/array/mod.rs
+++ b/src/common/src/array/mod.rs
@@ -35,7 +35,6 @@ mod stream_chunk_iter;
 pub mod stream_record;
 pub mod struct_array;
 mod utf8_array;
-mod value_reader;
 
 use std::convert::From;
 use std::hash::{Hash, Hasher};
@@ -65,7 +64,7 @@ pub use utf8_array::*;
 
 pub use self::error::ArrayError;
 pub use crate::array::num256_array::{Int256Array, Int256ArrayBuilder};
-use crate::buffer::Bitmap;
+use crate::bitmap::Bitmap;
 use crate::types::*;
 use crate::{dispatch_array_builder_variants, dispatch_array_variants, for_all_array_variants};
 pub type ArrayResult<T> = Result<T, ArrayError>;
@@ -706,7 +705,7 @@ mod test_util {
     use std::hash::{BuildHasher, Hasher};
 
     use super::Array;
-    use crate::buffer::Bitmap;
+    use crate::bitmap::Bitmap;
     use crate::util::iter_util::ZipEqFast;
 
     pub fn hash_finish<H: Hasher>(hashers: &[H]) -> Vec<u64> {
diff --git a/src/common/src/array/num256_array.rs b/src/common/src/array/num256_array.rs
index 48ee99ac21db2..ad8611a652e2c 100644
--- a/src/common/src/array/num256_array.rs
+++ b/src/common/src/array/num256_array.rs
@@ -21,7 +21,7 @@ use risingwave_pb::common::Buffer;
 use risingwave_pb::data::PbArray;
 
 use crate::array::{Array, ArrayBuilder, ArrayImpl, ArrayResult};
-use crate::buffer::{Bitmap, BitmapBuilder};
+use crate::bitmap::{Bitmap, BitmapBuilder};
 use crate::types::{DataType, Int256, Int256Ref, Scalar};
 
 #[derive(Debug, Clone, EstimateSize)]
diff --git a/src/common/src/array/primitive_array.rs b/src/common/src/array/primitive_array.rs
index 29c7bfb49289a..45e1bc0ddd847 100644
--- a/src/common/src/array/primitive_array.rs
+++ b/src/common/src/array/primitive_array.rs
@@ -13,16 +13,18 @@
 // limitations under the License.
 
 use std::fmt::Debug;
-use std::io::Write;
+use std::io::{Cursor, Write};
 use std::mem::size_of;
 
+use anyhow::Context;
+use byteorder::{BigEndian, ReadBytesExt};
 use risingwave_common_estimate_size::{EstimateSize, ZeroHeapSize};
 use risingwave_pb::common::buffer::CompressionType;
 use risingwave_pb::common::Buffer;
 use risingwave_pb::data::{ArrayType, PbArray};
 
 use super::{Array, ArrayBuilder, ArrayImpl, ArrayResult};
-use crate::buffer::{Bitmap, BitmapBuilder};
+use crate::bitmap::{Bitmap, BitmapBuilder};
 use crate::for_all_native_types;
 use crate::types::*;
 
@@ -50,6 +52,7 @@ where
 
     // item methods
     fn to_protobuf<T: Write>(self, output: &mut T) -> ArrayResult<usize>;
+    fn from_protobuf(cur: &mut Cursor<&[u8]>) -> ArrayResult<Self>;
 }
 
 macro_rules! impl_array_methods {
@@ -81,7 +84,7 @@ macro_rules! impl_array_methods {
 }
 
 macro_rules! impl_primitive_for_native_types {
-    ($({ $naive_type:ty, $scalar_type:ident } ),*) => {
+    ($({ $naive_type:ty, $scalar_type:ident, $read_fn:ident } ),*) => {
         $(
             impl PrimitiveArrayItemType for $naive_type {
                 impl_array_methods!($naive_type, $scalar_type, $scalar_type);
@@ -89,6 +92,12 @@ macro_rules! impl_primitive_for_native_types {
                 fn to_protobuf<T: Write>(self, output: &mut T) -> ArrayResult<usize> {
                     NativeType::to_protobuf(self, output)
                 }
+                fn from_protobuf(cur: &mut Cursor<&[u8]>) -> ArrayResult<Self> {
+                    let v = cur
+                        .$read_fn::<BigEndian>()
+                        .context("failed to read value from buffer")?;
+                    Ok(v.into())
+                }
             }
         )*
     }
@@ -106,6 +115,9 @@ macro_rules! impl_primitive_for_others {
                 fn to_protobuf<T: Write>(self, output: &mut T) -> ArrayResult<usize> {
                     <$scalar_type>::to_protobuf(self, output)
                 }
+                fn from_protobuf(cur: &mut Cursor<&[u8]>) -> ArrayResult<Self> {
+                    <$scalar_type>::from_protobuf(cur)
+                }
             }
         )*
     }
diff --git a/src/common/src/array/proto_reader.rs b/src/common/src/array/proto_reader.rs
index 073ad0b3de7ba..3238368ed1041 100644
--- a/src/common/src/array/proto_reader.rs
+++ b/src/common/src/array/proto_reader.rs
@@ -16,43 +16,32 @@ use std::io::{Cursor, Read};
 
 use anyhow::Context;
 use byteorder::{BigEndian, ReadBytesExt};
-use paste::paste;
 use risingwave_pb::data::PbArrayType;
 
 use super::*;
-use crate::array::value_reader::{PrimitiveValueReader, VarSizedValueReader};
 
 impl ArrayImpl {
     pub fn from_protobuf(array: &PbArray, cardinality: usize) -> ArrayResult<Self> {
-        use crate::array::value_reader::*;
         let array = match array.array_type() {
-            PbArrayType::Int16 => read_numeric_array::<i16, I16ValueReader>(array, cardinality)?,
-            PbArrayType::Int32 => read_numeric_array::<i32, I32ValueReader>(array, cardinality)?,
-            PbArrayType::Int64 => read_numeric_array::<i64, I64ValueReader>(array, cardinality)?,
-            PbArrayType::Serial => {
-                read_numeric_array::<Serial, SerialValueReader>(array, cardinality)?
-            }
-            PbArrayType::Float32 => read_numeric_array::<F32, F32ValueReader>(array, cardinality)?,
-            PbArrayType::Float64 => read_numeric_array::<F64, F64ValueReader>(array, cardinality)?,
+            PbArrayType::Unspecified => unreachable!(),
+            PbArrayType::Int16 => read_primitive_array::<i16>(array, cardinality)?,
+            PbArrayType::Int32 => read_primitive_array::<i32>(array, cardinality)?,
+            PbArrayType::Int64 => read_primitive_array::<i64>(array, cardinality)?,
+            PbArrayType::Serial => read_primitive_array::<Serial>(array, cardinality)?,
+            PbArrayType::Float32 => read_primitive_array::<F32>(array, cardinality)?,
+            PbArrayType::Float64 => read_primitive_array::<F64>(array, cardinality)?,
             PbArrayType::Bool => read_bool_array(array, cardinality)?,
-            PbArrayType::Utf8 => {
-                read_string_array::<Utf8ArrayBuilder, Utf8ValueReader>(array, cardinality)?
-            }
-            PbArrayType::Decimal => {
-                read_numeric_array::<Decimal, DecimalValueReader>(array, cardinality)?
-            }
-            PbArrayType::Date => read_date_array(array, cardinality)?,
-            PbArrayType::Time => read_time_array(array, cardinality)?,
-            PbArrayType::Timestamp => read_timestamp_array(array, cardinality)?,
-            PbArrayType::Timestamptz => read_timestamptz_array(array, cardinality)?,
-            PbArrayType::Interval => read_interval_array(array, cardinality)?,
+            PbArrayType::Utf8 => read_string_array::<Utf8ValueReader>(array, cardinality)?,
+            PbArrayType::Decimal => read_primitive_array::<Decimal>(array, cardinality)?,
+            PbArrayType::Date => read_primitive_array::<Date>(array, cardinality)?,
+            PbArrayType::Time => read_primitive_array::<Time>(array, cardinality)?,
+            PbArrayType::Timestamp => read_primitive_array::<Timestamp>(array, cardinality)?,
+            PbArrayType::Timestamptz => read_primitive_array::<Timestamptz>(array, cardinality)?,
+            PbArrayType::Interval => read_primitive_array::<Interval>(array, cardinality)?,
             PbArrayType::Jsonb => JsonbArray::from_protobuf(array)?,
             PbArrayType::Struct => StructArray::from_protobuf(array)?,
             PbArrayType::List => ListArray::from_protobuf(array)?,
-            PbArrayType::Unspecified => unreachable!(),
-            PbArrayType::Bytea => {
-                read_string_array::<BytesArrayBuilder, BytesValueReader>(array, cardinality)?
-            }
+            PbArrayType::Bytea => read_string_array::<BytesValueReader>(array, cardinality)?,
             PbArrayType::Int256 => Int256Array::from_protobuf(array, cardinality)?,
         };
         Ok(array)
@@ -62,7 +51,7 @@ impl ArrayImpl {
 // TODO: Use techniques like apache arrow flight RPC to eliminate deserialization.
 // https://arrow.apache.org/docs/format/Flight.html
 
-fn read_numeric_array<T: PrimitiveArrayItemType, R: PrimitiveValueReader<T>>(
+fn read_primitive_array<T: PrimitiveArrayItemType>(
     array: &PbArray,
     cardinality: usize,
 ) -> ArrayResult<ArrayImpl> {
@@ -78,7 +67,7 @@ fn read_numeric_array<T: PrimitiveArrayItemType, R: PrimitiveValueReader<T>>(
     let mut cursor = Cursor::new(buf);
     for not_null in bitmap.iter() {
         if not_null {
-            let v = R::read(&mut cursor)?;
+            let v = T::from_protobuf(&mut cursor)?;
             builder.append(Some(v));
         } else {
             builder.append(None);
@@ -105,99 +94,52 @@ fn read_bool_array(array: &PbArray, cardinality: usize) -> ArrayResult<ArrayImpl
     Ok(arr.into())
 }
 
-fn read_date(cursor: &mut Cursor<&[u8]>) -> ArrayResult<Date> {
-    let days = cursor
-        .read_i32::<BigEndian>()
-        .context("failed to read i32 from Date buffer")?;
-
-    Ok(Date::with_days(days)?)
-}
-
-fn read_time(cursor: &mut Cursor<&[u8]>) -> ArrayResult<Time> {
-    let nano = cursor
-        .read_u64::<BigEndian>()
-        .context("failed to read u64 from Time buffer")?;
-
-    Ok(Time::with_nano(nano)?)
-}
-
-fn read_timestamp(cursor: &mut Cursor<&[u8]>) -> ArrayResult<Timestamp> {
-    let micros = cursor
+fn read_offset(offset_cursor: &mut Cursor<&[u8]>) -> ArrayResult<i64> {
+    let offset = offset_cursor
         .read_i64::<BigEndian>()
-        .context("failed to read i64 from Timestamp buffer")?;
+        .context("failed to read i64 from offset buffer")?;
 
-    Ok(Timestamp::with_micros(micros)?)
+    Ok(offset)
 }
 
-fn read_timestamptz(cursor: &mut Cursor<&[u8]>) -> ArrayResult<Timestamptz> {
-    let micros = cursor
-        .read_i64::<BigEndian>()
-        .context("failed to read i64 from Timestamptz buffer")?;
-
-    Timestamptz::from_protobuf(micros)
+trait VarSizedValueReader {
+    type AB: ArrayBuilder;
+    fn new_builder(capacity: usize) -> Self::AB;
+    fn read(buf: &[u8], builder: &mut Self::AB) -> ArrayResult<()>;
 }
 
-fn read_interval(cursor: &mut Cursor<&[u8]>) -> ArrayResult<Interval> {
-    let mut read = || {
-        let months = cursor.read_i32::<BigEndian>()?;
-        let days = cursor.read_i32::<BigEndian>()?;
-        let usecs = cursor.read_i64::<BigEndian>()?;
-
-        Ok::<_, std::io::Error>(Interval::from_month_day_usec(months, days, usecs))
-    };
+struct Utf8ValueReader;
 
-    Ok(read().context("failed to read Interval from buffer")?)
-}
+impl VarSizedValueReader for Utf8ValueReader {
+    type AB = Utf8ArrayBuilder;
 
-macro_rules! read_one_value_array {
-    ($({ $type:ident, $builder:ty }),*) => {
-        paste! {
-            $(
-            fn [<read_ $type:snake _array>](array: &PbArray, cardinality: usize) -> ArrayResult<ArrayImpl> {
-                ensure!(
-                    array.get_values().len() == 1,
-                    "Must have only 1 buffer in a {} array", stringify!($type)
-                );
+    fn new_builder(capacity: usize) -> Self::AB {
+        Utf8ArrayBuilder::new(capacity)
+    }
 
-                let buf = array.get_values()[0].get_body().as_slice();
-
-                let mut builder = $builder::new(cardinality);
-                let bitmap: Bitmap = array.get_null_bitmap()?.into();
-                let mut cursor = Cursor::new(buf);
-                for not_null in bitmap.iter() {
-                    if not_null {
-                        builder.append(Some([<read_ $type:snake>](&mut cursor)?));
-                    } else {
-                        builder.append(None);
-                    }
-                }
-                let arr = builder.finish();
-                ensure_eq!(arr.len(), cardinality);
-
-                Ok(arr.into())
-            }
-            )*
-        }
-    };
+    fn read(buf: &[u8], builder: &mut Utf8ArrayBuilder) -> ArrayResult<()> {
+        let s = std::str::from_utf8(buf).context("failed to read utf8 string from bytes")?;
+        builder.append(Some(s));
+        Ok(())
+    }
 }
 
-read_one_value_array! {
-    { Interval, IntervalArrayBuilder },
-    { Date, DateArrayBuilder },
-    { Time, TimeArrayBuilder },
-    { Timestamp, TimestampArrayBuilder },
-    { Timestamptz, TimestamptzArrayBuilder }
-}
+struct BytesValueReader;
 
-fn read_offset(offset_cursor: &mut Cursor<&[u8]>) -> ArrayResult<i64> {
-    let offset = offset_cursor
-        .read_i64::<BigEndian>()
-        .context("failed to read i64 from offset buffer")?;
+impl VarSizedValueReader for BytesValueReader {
+    type AB = BytesArrayBuilder;
 
-    Ok(offset)
+    fn new_builder(capacity: usize) -> Self::AB {
+        BytesArrayBuilder::new(capacity)
+    }
+
+    fn read(buf: &[u8], builder: &mut BytesArrayBuilder) -> ArrayResult<()> {
+        builder.append(Some(buf));
+        Ok(())
+    }
 }
 
-fn read_string_array<B: ArrayBuilder, R: VarSizedValueReader<B>>(
+fn read_string_array<R: VarSizedValueReader>(
     array: &PbArray,
     cardinality: usize,
 ) -> ArrayResult<ArrayImpl> {
@@ -208,7 +150,7 @@ fn read_string_array<B: ArrayBuilder, R: VarSizedValueReader<B>>(
     let offset_buff = array.get_values()[0].get_body().as_slice();
     let data_buf = array.get_values()[1].get_body().as_slice();
 
-    let mut builder = B::new(cardinality);
+    let mut builder = R::new_builder(cardinality);
     let bitmap: Bitmap = array.get_null_bitmap()?.into();
     let mut offset_cursor = Cursor::new(offset_buff);
     let mut data_cursor = Cursor::new(data_buf);
@@ -348,7 +290,7 @@ mod tests {
         let mut builder = DateArrayBuilder::new(cardinality);
         for i in 0..cardinality {
             if i % 2 == 0 {
-                builder.append(Date::with_days(i as i32).ok());
+                builder.append(Date::with_days_since_ce(i as i32).ok());
             } else {
                 builder.append(None);
             }
@@ -359,7 +301,7 @@ mod tests {
         let arr: &DateArray = new_col.as_date();
         arr.iter().enumerate().for_each(|(i, x)| {
             if i % 2 == 0 {
-                assert_eq!(Date::with_days(i as i32).ok().unwrap(), x.unwrap());
+                assert_eq!(Date::with_days_since_ce(i as i32).ok().unwrap(), x.unwrap());
             } else {
                 assert!(x.is_none());
             }
diff --git a/src/common/src/array/stream_chunk.rs b/src/common/src/array/stream_chunk.rs
index e3776de6d0fbc..cfee990561aa1 100644
--- a/src/common/src/array/stream_chunk.rs
+++ b/src/common/src/array/stream_chunk.rs
@@ -29,7 +29,7 @@ use risingwave_pb::data::{PbOp, PbStreamChunk};
 use super::stream_chunk_builder::StreamChunkBuilder;
 use super::{ArrayImpl, ArrayRef, ArrayResult, DataChunkTestExt, RowRef};
 use crate::array::DataChunk;
-use crate::buffer::{Bitmap, BitmapBuilder};
+use crate::bitmap::{Bitmap, BitmapBuilder};
 use crate::catalog::Schema;
 use crate::field_generator::VarcharProperty;
 use crate::row::Row;
@@ -525,6 +525,10 @@ impl OpRowMutRef<'_> {
 }
 
 impl StreamChunkMut {
+    pub fn capacity(&self) -> usize {
+        self.vis.len()
+    }
+
     pub fn vis(&self, i: usize) -> bool {
         self.vis.is_set(i)
     }
diff --git a/src/common/src/array/stream_chunk_builder.rs b/src/common/src/array/stream_chunk_builder.rs
index c44d313fffffe..5fd6ae8c761c1 100644
--- a/src/common/src/array/stream_chunk_builder.rs
+++ b/src/common/src/array/stream_chunk_builder.rs
@@ -14,7 +14,7 @@
 
 use crate::array::stream_record::Record;
 use crate::array::{ArrayBuilderImpl, Op, StreamChunk};
-use crate::buffer::BitmapBuilder;
+use crate::bitmap::BitmapBuilder;
 use crate::row::Row;
 use crate::types::{DataType, DatumRef};
 use crate::util::iter_util::ZipEqFast;
diff --git a/src/common/src/array/struct_array.rs b/src/common/src/array/struct_array.rs
index e5a1687bbcfeb..22aae00c84f4c 100644
--- a/src/common/src/array/struct_array.rs
+++ b/src/common/src/array/struct_array.rs
@@ -25,7 +25,7 @@ use risingwave_pb::data::{PbArray, PbArrayType, StructArrayData};
 
 use super::{Array, ArrayBuilder, ArrayBuilderImpl, ArrayImpl, ArrayResult, DataChunk};
 use crate::array::ArrayRef;
-use crate::buffer::{Bitmap, BitmapBuilder};
+use crate::bitmap::{Bitmap, BitmapBuilder};
 use crate::error::BoxedError;
 use crate::types::{
     hash_datum, DataType, Datum, DatumRef, DefaultOrd, Scalar, ScalarImpl, StructType, ToDatumRef,
diff --git a/src/common/src/array/utf8_array.rs b/src/common/src/array/utf8_array.rs
index 5580ab6cc5970..8463f73e3ee58 100644
--- a/src/common/src/array/utf8_array.rs
+++ b/src/common/src/array/utf8_array.rs
@@ -19,7 +19,7 @@ use risingwave_pb::data::{ArrayType, PbArray};
 
 use super::bytes_array::{BytesWriter, PartialBytesWriter};
 use super::{Array, ArrayBuilder, BytesArray, BytesArrayBuilder, DataType};
-use crate::buffer::Bitmap;
+use crate::bitmap::Bitmap;
 
 /// `Utf8Array` is a collection of Rust Utf8 `str`s. It's a wrapper of `BytesArray`.
 #[derive(Debug, Clone, PartialEq, Eq)]
diff --git a/src/common/src/array/value_reader.rs b/src/common/src/array/value_reader.rs
deleted file mode 100644
index b418a0c315adf..0000000000000
--- a/src/common/src/array/value_reader.rs
+++ /dev/null
@@ -1,88 +0,0 @@
-// Copyright 2024 RisingWave Labs
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::io::Cursor;
-use std::str::from_utf8;
-
-use anyhow::Context;
-use byteorder::{BigEndian, ReadBytesExt};
-
-use super::ArrayResult;
-use crate::array::{
-    ArrayBuilder, BytesArrayBuilder, PrimitiveArrayItemType, Serial, Utf8ArrayBuilder,
-};
-use crate::types::{Decimal, F32, F64};
-
-/// Reads an encoded buffer into a value.
-pub trait PrimitiveValueReader<T: PrimitiveArrayItemType> {
-    fn read(cur: &mut Cursor<&[u8]>) -> ArrayResult<T>;
-}
-
-pub struct I16ValueReader;
-pub struct I32ValueReader;
-pub struct I64ValueReader;
-pub struct SerialValueReader;
-pub struct F32ValueReader;
-pub struct F64ValueReader;
-
-macro_rules! impl_numeric_value_reader {
-    ($value_type:ty, $value_reader:ty, $read_fn:ident) => {
-        impl PrimitiveValueReader<$value_type> for $value_reader {
-            fn read(cur: &mut Cursor<&[u8]>) -> ArrayResult<$value_type> {
-                let v = cur
-                    .$read_fn::<BigEndian>()
-                    .context("failed to read value from buffer")?;
-                Ok(v.into())
-            }
-        }
-    };
-}
-
-impl_numeric_value_reader!(i16, I16ValueReader, read_i16);
-impl_numeric_value_reader!(i32, I32ValueReader, read_i32);
-impl_numeric_value_reader!(i64, I64ValueReader, read_i64);
-impl_numeric_value_reader!(Serial, SerialValueReader, read_i64);
-impl_numeric_value_reader!(F32, F32ValueReader, read_f32);
-impl_numeric_value_reader!(F64, F64ValueReader, read_f64);
-
-pub struct DecimalValueReader;
-
-impl PrimitiveValueReader<Decimal> for DecimalValueReader {
-    fn read(cur: &mut Cursor<&[u8]>) -> ArrayResult<Decimal> {
-        Decimal::from_protobuf(cur)
-    }
-}
-
-pub trait VarSizedValueReader<AB: ArrayBuilder> {
-    fn read(buf: &[u8], builder: &mut AB) -> ArrayResult<()>;
-}
-
-pub struct Utf8ValueReader;
-
-impl VarSizedValueReader<Utf8ArrayBuilder> for Utf8ValueReader {
-    fn read(buf: &[u8], builder: &mut Utf8ArrayBuilder) -> ArrayResult<()> {
-        let s = from_utf8(buf).context("failed to read utf8 string from bytes")?;
-        builder.append(Some(s));
-        Ok(())
-    }
-}
-
-pub struct BytesValueReader;
-
-impl VarSizedValueReader<BytesArrayBuilder> for BytesValueReader {
-    fn read(buf: &[u8], builder: &mut BytesArrayBuilder) -> ArrayResult<()> {
-        builder.append(Some(buf));
-        Ok(())
-    }
-}
diff --git a/src/common/src/buffer/bitmap.rs b/src/common/src/bitmap.rs
similarity index 99%
rename from src/common/src/buffer/bitmap.rs
rename to src/common/src/bitmap.rs
index c6a11da093ed4..7ef6bf039f47d 100644
--- a/src/common/src/buffer/bitmap.rs
+++ b/src/common/src/bitmap.rs
@@ -425,7 +425,7 @@ impl Bitmap {
     ///
     /// # Example
     /// ```
-    /// use risingwave_common::buffer::Bitmap;
+    /// use risingwave_common::bitmap::Bitmap;
     /// let bitmap = Bitmap::from_range(200, 100..180);
     /// assert_eq!(bitmap.count_ones(), 80);
     /// for i in 0..200 {
diff --git a/src/common/src/buffer/mod.rs b/src/common/src/buffer/mod.rs
deleted file mode 100644
index 7628f6eddceb4..0000000000000
--- a/src/common/src/buffer/mod.rs
+++ /dev/null
@@ -1,33 +0,0 @@
-// Copyright 2024 RisingWave Labs
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-mod bitmap;
-pub use bitmap::*;
diff --git a/src/common/src/catalog/external_table.rs b/src/common/src/catalog/external_table.rs
index 797d789b85fc0..adc49cac4da97 100644
--- a/src/common/src/catalog/external_table.rs
+++ b/src/common/src/catalog/external_table.rs
@@ -15,6 +15,7 @@
 use std::collections::{BTreeMap, HashMap};
 
 use risingwave_pb::plan_common::ExternalTableDesc;
+use risingwave_pb::secret::PbSecretRef;
 
 use super::{ColumnDesc, ColumnId, TableId};
 use crate::util::sort_util::ColumnOrder;
@@ -42,6 +43,8 @@ pub struct CdcTableDesc {
 
     /// properties will be passed into the `StreamScanNode`
     pub connect_properties: BTreeMap<String, String>,
+    /// Secret refs
+    pub secret_refs: BTreeMap<String, PbSecretRef>,
 }
 
 impl CdcTableDesc {
@@ -65,7 +68,7 @@ impl CdcTableDesc {
             table_name: self.external_table_name.clone(),
             stream_key: self.stream_key.iter().map(|k| *k as _).collect(),
             connect_properties: self.connect_properties.clone(),
-            secret_refs: Default::default(),
+            secret_refs: self.secret_refs.clone(),
         }
     }
 
diff --git a/src/common/src/catalog/schema.rs b/src/common/src/catalog/schema.rs
index 113d9f804b3d4..9eccecfc2fc00 100644
--- a/src/common/src/catalog/schema.rs
+++ b/src/common/src/catalog/schema.rs
@@ -197,6 +197,14 @@ impl Schema {
             true
         }
     }
+
+    pub fn formatted_col_names(&self) -> String {
+        self.fields
+            .iter()
+            .map(|f| format!("\"{}\"", &f.name))
+            .collect::<Vec<_>>()
+            .join(", ")
+    }
 }
 
 impl Field {
diff --git a/src/common/src/config.rs b/src/common/src/config.rs
index 47d53a75b76a1..eeb393edb4ab9 100644
--- a/src/common/src/config.rs
+++ b/src/common/src/config.rs
@@ -223,6 +223,12 @@ pub struct MetaConfig {
     #[serde(default = "default::meta::enable_hummock_data_archive")]
     pub enable_hummock_data_archive: bool,
 
+    /// The interval at which a Hummock version snapshot is taken for time travel.
+    ///
+    /// Larger value indicates less storage overhead but worse query performance.
+    #[serde(default = "default::meta::hummock_time_travel_snapshot_interval")]
+    pub hummock_time_travel_snapshot_interval: u64,
+
     /// The minimum delta log number a new checkpoint should compact, otherwise the checkpoint
     /// attempt is rejected.
     #[serde(default = "default::meta::min_delta_log_num_for_hummock_version_checkpoint")]
@@ -369,9 +375,6 @@ pub struct MetaConfig {
     /// Whether compactor should rewrite row to remove dropped column.
     #[serde(default = "default::meta::enable_dropped_column_reclaim")]
     pub enable_dropped_column_reclaim: bool,
-
-    #[serde(default = "default::meta::secret_store_private_key")]
-    pub secret_store_private_key: Vec<u8>,
 }
 
 #[derive(Copy, Clone, Debug, Default)]
@@ -472,8 +475,14 @@ pub struct ServerConfig {
     #[serde(default = "default::server::heartbeat_interval_ms")]
     pub heartbeat_interval_ms: u32,
 
+    /// The default number of the connections when connecting to a gRPC server.
+    ///
+    /// For the connections used in streaming or batch exchange, please refer to the entries in
+    /// `[stream.developer]` and `[batch.developer]` sections. This value will be used if they
+    /// are not specified.
     #[serde(default = "default::server::connection_pool_size")]
-    pub connection_pool_size: u16,
+    // Intentionally made private to avoid abuse. Check the related methods on `RwConfig`.
+    connection_pool_size: u16,
 
     /// Used for control the metrics level, similar to log level.
     #[serde(default = "default::server::metrics_level")]
@@ -851,6 +860,9 @@ pub struct FileCacheConfig {
     #[serde(default = "default::file_cache::compression")]
     pub compression: String,
 
+    #[serde(default = "default::file_cache::flush_buffer_threshold_mb")]
+    pub flush_buffer_threshold_mb: Option<usize>,
+
     #[serde(default, flatten)]
     #[config_doc(omitted)]
     pub unrecognized: Unrecognized<Self>,
@@ -998,6 +1010,11 @@ pub struct StreamingDeveloperConfig {
     /// Actor tokio metrics is enabled if `enable_actor_tokio_metrics` is set or metrics level >= Debug.
     #[serde(default = "default::developer::enable_actor_tokio_metrics")]
     pub enable_actor_tokio_metrics: bool,
+
+    /// The number of the connections for streaming remote exchange between two nodes.
+    /// If not specified, the value of `server.connection_pool_size` will be used.
+    #[serde(default = "default::developer::stream_exchange_connection_pool_size")]
+    pub exchange_connection_pool_size: Option<u16>,
 }
 
 /// The subsections `[batch.developer]`.
@@ -1017,6 +1034,11 @@ pub struct BatchDeveloperConfig {
     /// The size of a chunk produced by `RowSeqScanExecutor`
     #[serde(default = "default::developer::batch_chunk_size")]
     pub chunk_size: usize,
+
+    /// The number of the connections for batch remote exchange between two nodes.
+    /// If not specified, the value of `server.connection_pool_size` will be used.
+    #[serde(default = "default::developer::batch_exchange_connection_pool_size")]
+    exchange_connection_pool_size: Option<u16>,
 }
 
 macro_rules! define_system_config {
@@ -1066,6 +1088,9 @@ pub struct ObjectStoreConfig {
     // TODO: the following field will be deprecated after opendal is stablized
     #[serde(default)]
     pub opendal_writer_abort_on_err: bool,
+
+    #[serde(default = "default::object_store_config::upload_part_size")]
+    pub upload_part_size: usize,
 }
 
 impl ObjectStoreConfig {
@@ -1253,6 +1278,30 @@ impl SystemConfig {
     }
 }
 
+impl RwConfig {
+    pub const fn default_connection_pool_size(&self) -> u16 {
+        self.server.connection_pool_size
+    }
+
+    /// Returns [`StreamingDeveloperConfig::exchange_connection_pool_size`] if set,
+    /// otherwise [`ServerConfig::connection_pool_size`].
+    pub fn streaming_exchange_connection_pool_size(&self) -> u16 {
+        self.streaming
+            .developer
+            .exchange_connection_pool_size
+            .unwrap_or_else(|| self.default_connection_pool_size())
+    }
+
+    /// Returns [`BatchDeveloperConfig::exchange_connection_pool_size`] if set,
+    /// otherwise [`ServerConfig::connection_pool_size`].
+    pub fn batch_exchange_connection_pool_size(&self) -> u16 {
+        self.batch
+            .developer
+            .exchange_connection_pool_size
+            .unwrap_or_else(|| self.default_connection_pool_size())
+    }
+}
+
 pub mod default {
     pub mod meta {
         use crate::config::{DefaultParallelism, MetaBackend};
@@ -1278,7 +1327,7 @@ pub mod default {
         }
 
         pub fn vacuum_spin_interval_ms() -> u64 {
-            10
+            200
         }
 
         pub fn hummock_version_checkpoint_interval_sec() -> u64 {
@@ -1289,6 +1338,10 @@ pub mod default {
             false
         }
 
+        pub fn hummock_time_travel_snapshot_interval() -> u64 {
+            100
+        }
+
         pub fn min_delta_log_num_for_hummock_version_checkpoint() -> u64 {
             10
         }
@@ -1396,10 +1449,6 @@ pub mod default {
         pub fn enable_dropped_column_reclaim() -> bool {
             false
         }
-
-        pub fn secret_store_private_key() -> Vec<u8> {
-            "demo-secret-private-key".as_bytes().to_vec()
-        }
     }
 
     pub mod server {
@@ -1591,6 +1640,14 @@ pub mod default {
         pub fn table_info_statistic_history_times() -> usize {
             240
         }
+
+        pub fn block_file_cache_flush_buffer_threshold_mb() -> usize {
+            256
+        }
+
+        pub fn meta_file_cache_flush_buffer_threshold_mb() -> usize {
+            64
+        }
     }
 
     pub mod streaming {
@@ -1651,6 +1708,10 @@ pub mod default {
         pub fn compression() -> String {
             "none".to_string()
         }
+
+        pub fn flush_buffer_threshold_mb() -> Option<usize> {
+            None
+        }
     }
 
     pub mod cache_refill {
@@ -1714,6 +1775,12 @@ pub mod default {
             1024
         }
 
+        /// Default to unset to be compatible with the behavior before this config is introduced,
+        /// that is, follow the value of `server.connection_pool_size`.
+        pub fn batch_exchange_connection_pool_size() -> Option<u16> {
+            None
+        }
+
         pub fn stream_enable_executor_row_count() -> bool {
             false
         }
@@ -1810,6 +1877,11 @@ pub mod default {
             2048
         }
 
+        /// Default to 1 to be compatible with the behavior before this config is introduced.
+        pub fn stream_exchange_connection_pool_size() -> Option<u16> {
+            Some(1)
+        }
+
         pub fn enable_actor_tokio_metrics() -> bool {
             false
         }
@@ -2038,6 +2110,11 @@ pub mod default {
             8
         }
 
+        pub fn upload_part_size() -> usize {
+            // 16m
+            16 * 1024 * 1024
+        }
+
         pub mod s3 {
             const DEFAULT_IDENTITY_RESOLUTION_TIMEOUT_S: u64 = 5;
 
@@ -2123,6 +2200,8 @@ pub struct StorageMemoryConfig {
     pub prefetch_buffer_capacity_mb: usize,
     pub block_cache_eviction_config: EvictionConfig,
     pub meta_cache_eviction_config: EvictionConfig,
+    pub block_file_cache_flush_buffer_threshold_mb: usize,
+    pub meta_file_cache_flush_buffer_threshold_mb: usize,
 }
 
 pub const MAX_META_CACHE_SHARD_BITS: usize = 4;
@@ -2234,6 +2313,17 @@ pub fn extract_storage_memory_config(s: &RwConfig) -> StorageMemoryConfig {
                 }
             });
 
+    let block_file_cache_flush_buffer_threshold_mb = s
+        .storage
+        .data_file_cache
+        .flush_buffer_threshold_mb
+        .unwrap_or(default::storage::block_file_cache_flush_buffer_threshold_mb());
+    let meta_file_cache_flush_buffer_threshold_mb = s
+        .storage
+        .meta_file_cache
+        .flush_buffer_threshold_mb
+        .unwrap_or(default::storage::block_file_cache_flush_buffer_threshold_mb());
+
     StorageMemoryConfig {
         block_cache_capacity_mb,
         block_cache_shard_num,
@@ -2244,6 +2334,8 @@ pub fn extract_storage_memory_config(s: &RwConfig) -> StorageMemoryConfig {
         prefetch_buffer_capacity_mb,
         block_cache_eviction_config,
         meta_cache_eviction_config,
+        block_file_cache_flush_buffer_threshold_mb,
+        meta_file_cache_flush_buffer_threshold_mb,
     }
 }
 
diff --git a/src/common/src/hash/consistent_hash/bitmap.rs b/src/common/src/hash/consistent_hash/bitmap.rs
index f6e55f3ebec50..773231ba36a89 100644
--- a/src/common/src/hash/consistent_hash/bitmap.rs
+++ b/src/common/src/hash/consistent_hash/bitmap.rs
@@ -14,7 +14,7 @@
 
 use std::ops::RangeInclusive;
 
-use crate::buffer::Bitmap;
+use crate::bitmap::Bitmap;
 use crate::hash::VirtualNode;
 
 /// An extension trait for `Bitmap` to support virtual node operations.
diff --git a/src/common/src/hash/consistent_hash/mapping.rs b/src/common/src/hash/consistent_hash/mapping.rs
index 59460e7966220..a462acb291853 100644
--- a/src/common/src/hash/consistent_hash/mapping.rs
+++ b/src/common/src/hash/consistent_hash/mapping.rs
@@ -12,28 +12,26 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::{BTreeSet, HashMap, HashSet};
+use std::collections::{BTreeSet, HashMap};
 use std::fmt::{Debug, Display, Formatter};
 use std::hash::Hash;
-use std::ops::{Index, Sub};
+use std::ops::Index;
 
 use educe::Educe;
 use itertools::Itertools;
-use risingwave_pb::common::{
-    ParallelUnit, ParallelUnitMapping as ParallelUnitMappingProto, PbWorkerSlotMapping,
-};
+use risingwave_pb::common::PbWorkerSlotMapping;
 use risingwave_pb::stream_plan::ActorMapping as ActorMappingProto;
 
 use super::bitmap::VnodeBitmapExt;
-use super::vnode::{ParallelUnitId, VirtualNode};
-use crate::buffer::{Bitmap, BitmapBuilder};
+use crate::bitmap::{Bitmap, BitmapBuilder};
+use crate::hash::VirtualNode;
 use crate::util::compress::compress_data;
 use crate::util::iter_util::ZipEqDebug;
 
 // TODO: find a better place for this.
 pub type ActorId = u32;
 
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Ord, PartialOrd)]
+#[derive(Clone, Copy, PartialEq, Eq, Hash, Ord, PartialOrd)]
 pub struct WorkerSlotId(u64);
 
 impl WorkerSlotId {
@@ -68,11 +66,17 @@ impl Display for WorkerSlotId {
     }
 }
 
+impl Debug for WorkerSlotId {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.write_fmt(format_args!("[{}:{}]", self.worker_id(), self.slot_idx()))
+    }
+}
+
 /// Trait for items that can be used as keys in [`VnodeMapping`].
 pub trait VnodeMappingItem {
     /// The type of the item.
     ///
-    /// Currently, there are two types of items: [`ParallelUnitId`] and [`ActorId`]. We don't use
+    /// Currently, there are two types of items: [`WorkerSlotId`] and [`ActorId`]. We don't use
     /// them directly as the generic parameter because they're the same type aliases.
     type Item: Copy + Ord + Hash + Debug;
 }
@@ -286,12 +290,6 @@ pub mod marker {
         type Item = ActorId;
     }
 
-    /// A marker type for items of [`ParallelUnitId`].
-    pub struct ParallelUnit;
-    impl VnodeMappingItem for ParallelUnit {
-        type Item = ParallelUnitId;
-    }
-
     /// A marker type for items of [`WorkerSlotId`].
     pub struct WorkerSlot;
     impl VnodeMappingItem for WorkerSlot {
@@ -304,23 +302,35 @@ pub type ActorMapping = VnodeMapping<marker::Actor>;
 /// An expanded mapping from [`VirtualNode`] to [`ActorId`].
 pub type ExpandedActorMapping = ExpandedMapping<marker::Actor>;
 
-/// A mapping from [`VirtualNode`] to [`ParallelUnitId`].
-pub type ParallelUnitMapping = VnodeMapping<marker::ParallelUnit>;
-/// An expanded mapping from [`VirtualNode`] to [`ParallelUnitId`].
-pub type ExpandedParallelUnitMapping = ExpandedMapping<marker::ParallelUnit>;
-
 /// A mapping from [`VirtualNode`] to [`WorkerSlotId`].
 pub type WorkerSlotMapping = VnodeMapping<marker::WorkerSlot>;
 /// An expanded mapping from [`VirtualNode`] to [`WorkerSlotId`].
 pub type ExpandedWorkerSlotMapping = ExpandedMapping<marker::WorkerSlot>;
 
 impl ActorMapping {
-    /// Transform this actor mapping to a parallel unit mapping, essentially `transform`.
-    pub fn to_parallel_unit<M>(&self, to_map: &M) -> ParallelUnitMapping
-    where
-        M: for<'a> Index<&'a ActorId, Output = ParallelUnitId>,
-    {
-        self.transform(to_map)
+    /// Transform the actor mapping to the worker slot mapping. Note that the parameter is a mapping from actor to worker.
+    pub fn to_worker_slot(&self, actor_to_worker: &HashMap<ActorId, u32>) -> WorkerSlotMapping {
+        let mut worker_actors = HashMap::new();
+        for actor_id in self.iter_unique() {
+            let worker_id = actor_to_worker
+                .get(&actor_id)
+                .cloned()
+                .unwrap_or_else(|| panic!("location for actor {} not found", actor_id));
+
+            worker_actors
+                .entry(worker_id)
+                .or_insert(BTreeSet::new())
+                .insert(actor_id);
+        }
+
+        let mut actor_location = HashMap::new();
+        for (worker, actors) in worker_actors {
+            for (idx, &actor) in actors.iter().enumerate() {
+                actor_location.insert(actor, WorkerSlotId::new(worker, idx));
+            }
+        }
+
+        self.transform(&actor_location)
     }
 
     /// Create an actor mapping from the protobuf representation.
@@ -341,12 +351,6 @@ impl ActorMapping {
     }
 }
 
-#[derive(thiserror::Error, Debug)]
-pub enum ParallelUnitError {
-    #[error("parallel units {0:?} are not covered by the worker slot mapping")]
-    NotCovered(HashSet<ParallelUnitId>),
-}
-
 impl WorkerSlotMapping {
     /// Create a uniform worker mapping from the given worker ids
     pub fn build_from_ids(worker_slot_ids: &[WorkerSlotId]) -> Self {
@@ -371,74 +375,11 @@ impl WorkerSlotMapping {
     }
 }
 
-impl ParallelUnitMapping {
-    /// Create a uniform parallel unit mapping from the given parallel units, essentially
-    /// `new_uniform`.
-    pub fn build(parallel_units: &[ParallelUnit]) -> Self {
-        Self::new_uniform(parallel_units.iter().map(|pu| pu.id))
-    }
-
-    /// Create a uniform parallel unit mapping from the given parallel units ids
-    pub fn build_from_ids(parallel_unit_ids: &[ParallelUnitId]) -> Self {
-        Self::new_uniform(parallel_unit_ids.iter().cloned())
-    }
-
-    /// Transform this parallel unit mapping to an actor mapping, essentially `transform`.
-    pub fn to_actor(&self, to_map: &HashMap<ParallelUnitId, ActorId>) -> ActorMapping {
+impl WorkerSlotMapping {
+    /// Transform this worker slot mapping to an actor mapping, essentially `transform`.
+    pub fn to_actor(&self, to_map: &HashMap<WorkerSlotId, ActorId>) -> ActorMapping {
         self.transform(to_map)
     }
-
-    /// Transform this parallel unit mapping to a worker slot mapping, essentially `transform`.
-    pub fn to_worker_slot(
-        &self,
-        to_map: &HashMap<ParallelUnitId, u32>,
-    ) -> Result<WorkerSlotMapping, ParallelUnitError> {
-        let mut worker_to_parallel_units = HashMap::<_, BTreeSet<_>>::new();
-        for (parallel_unit_id, worker_id) in to_map {
-            worker_to_parallel_units
-                .entry(*worker_id)
-                .or_default()
-                .insert(*parallel_unit_id);
-        }
-
-        let mut parallel_unit_to_worker_slot = HashMap::with_capacity(to_map.len());
-
-        for (worker_id, parallel_unit_ids) in worker_to_parallel_units {
-            for (index, &parallel_unit_id) in parallel_unit_ids.iter().enumerate() {
-                parallel_unit_to_worker_slot
-                    .insert(parallel_unit_id, WorkerSlotId::new(worker_id, index));
-            }
-        }
-
-        let available_parallel_unit_ids: HashSet<_> =
-            parallel_unit_to_worker_slot.keys().copied().collect();
-
-        let parallel_unit_ids: HashSet<_> = self.data.iter().copied().collect();
-
-        let sub_set = parallel_unit_ids.sub(&available_parallel_unit_ids);
-        if sub_set.is_empty() {
-            Ok(self.transform(&parallel_unit_to_worker_slot))
-        } else {
-            Err(ParallelUnitError::NotCovered(sub_set))
-        }
-    }
-
-    /// Create a parallel unit mapping from the protobuf representation.
-    pub fn from_protobuf(proto: &ParallelUnitMappingProto) -> Self {
-        assert_eq!(proto.original_indices.len(), proto.data.len());
-        Self {
-            original_indices: proto.original_indices.clone(),
-            data: proto.data.clone(),
-        }
-    }
-
-    /// Convert this parallel unit mapping to the protobuf representation.
-    pub fn to_protobuf(&self) -> ParallelUnitMappingProto {
-        ParallelUnitMappingProto {
-            original_indices: self.original_indices.clone(),
-            data: self.data.clone(),
-        }
-    }
 }
 
 #[cfg(test)]
diff --git a/src/common/src/hash/consistent_hash/vnode.rs b/src/common/src/hash/consistent_hash/vnode.rs
index fc6dac0978adf..f528544689f31 100644
--- a/src/common/src/hash/consistent_hash/vnode.rs
+++ b/src/common/src/hash/consistent_hash/vnode.rs
@@ -22,10 +22,6 @@ use crate::types::{DataType, Datum, DatumRef, ScalarImpl, ScalarRefImpl};
 use crate::util::hash_util::Crc32FastBuilder;
 use crate::util::row_id::extract_vnode_id_from_row_id;
 
-/// Parallel unit is the minimal scheduling unit.
-// TODO: make it a newtype
-pub type ParallelUnitId = u32;
-
 /// `VirtualNode` (a.k.a. Vnode) is a minimal partition that a set of keys belong to. It is used for
 /// consistent hashing.
 #[repr(transparent)]
diff --git a/src/common/src/hash/key.rs b/src/common/src/hash/key.rs
index e9f7e83ac9146..61a41e9a7365d 100644
--- a/src/common/src/hash/key.rs
+++ b/src/common/src/hash/key.rs
@@ -561,7 +561,7 @@ impl HashKeySer<'_> for Date {
 impl HashKeyDe for Date {
     fn deserialize(_data_type: &DataType, mut buf: impl Buf) -> Self {
         let days = buf.get_i32_ne();
-        Date::with_days(days).unwrap()
+        Date::with_days_since_ce(days).unwrap()
     }
 }
 
diff --git a/src/common/src/hash/table_distribution.rs b/src/common/src/hash/table_distribution.rs
index c69c9e9a43aec..9be9cd2abafb2 100644
--- a/src/common/src/hash/table_distribution.rs
+++ b/src/common/src/hash/table_distribution.rs
@@ -21,7 +21,7 @@ use risingwave_pb::plan_common::StorageTableDesc;
 use tracing::warn;
 
 use crate::array::{Array, DataChunk, PrimitiveArray};
-use crate::buffer::{Bitmap, BitmapBuilder};
+use crate::bitmap::{Bitmap, BitmapBuilder};
 use crate::hash::VirtualNode;
 use crate::row::Row;
 use crate::util::iter_util::ZipEqFast;
diff --git a/src/common/src/lib.rs b/src/common/src/lib.rs
index 05ca777ea7d39..8d47d0c621646 100644
--- a/src/common/src/lib.rs
+++ b/src/common/src/lib.rs
@@ -26,7 +26,6 @@
 #![feature(lint_reasons)]
 #![feature(coroutines)]
 #![feature(map_try_insert)]
-#![feature(lazy_cell)]
 #![feature(error_generic_member_access)]
 #![feature(let_chains)]
 #![feature(portable_simd)]
@@ -35,7 +34,6 @@
 #![allow(incomplete_features)]
 #![feature(iterator_try_collect)]
 #![feature(iter_order_by)]
-#![feature(exclusive_range_pattern)]
 #![feature(binary_heap_into_iter_sorted)]
 #![feature(impl_trait_in_assoc_type)]
 #![feature(map_entry_replace)]
@@ -65,7 +63,7 @@ pub mod array;
 #[macro_use]
 pub mod util;
 pub mod acl;
-pub mod buffer;
+pub mod bitmap;
 pub mod cache;
 pub mod cast;
 pub mod catalog;
@@ -80,7 +78,10 @@ pub use risingwave_common_metrics::{
     register_guarded_histogram_vec_with_registry, register_guarded_int_counter_vec_with_registry,
     register_guarded_int_gauge_vec_with_registry,
 };
-pub use {risingwave_common_metrics as metrics, risingwave_license as license};
+pub use {
+    risingwave_common_metrics as metrics, risingwave_common_secret as secret,
+    risingwave_license as license,
+};
 pub mod lru;
 pub mod opts;
 pub mod range;
diff --git a/src/common/src/session_config/mod.rs b/src/common/src/session_config/mod.rs
index 8fbfd2cd6e969..16cc337c6f46a 100644
--- a/src/common/src/session_config/mod.rs
+++ b/src/common/src/session_config/mod.rs
@@ -54,7 +54,8 @@ type SessionConfigResult<T> = std::result::Result<T, SessionConfigError>;
 
 // NOTE(kwannoel): We declare it separately as a constant,
 // otherwise seems like it can't infer the type of -1 when written inline.
-const DISABLE_STREAMING_RATE_LIMIT: i32 = -1;
+const DISABLE_BACKFILL_RATE_LIMIT: i32 = -1;
+const DISABLE_SOURCE_RATE_LIMIT: i32 = -1;
 
 #[serde_as]
 /// This is the Session Config of RisingWave.
@@ -253,11 +254,17 @@ pub struct SessionConfig {
     #[parameter(default = STANDARD_CONFORMING_STRINGS)]
     standard_conforming_strings: String,
 
+    /// Set streaming rate limit (rows per second) for each parallelism for mv / source / sink backfilling
+    /// If set to -1, disable rate limit.
+    /// If set to 0, this pauses the snapshot read / source read.
+    #[parameter(default = DISABLE_BACKFILL_RATE_LIMIT)]
+    backfill_rate_limit: i32,
+
     /// Set streaming rate limit (rows per second) for each parallelism for mv / source backfilling, source reads.
     /// If set to -1, disable rate limit.
     /// If set to 0, this pauses the snapshot read / source read.
-    #[parameter(default = DISABLE_STREAMING_RATE_LIMIT)]
-    streaming_rate_limit: i32,
+    #[parameter(default = DISABLE_SOURCE_RATE_LIMIT)]
+    source_rate_limit: i32,
 
     /// Cache policy for partition cache in streaming over window.
     /// Can be "full", "recent", "`recent_first_n`" or "`recent_last_n`".
diff --git a/src/common/src/system_param/mod.rs b/src/common/src/system_param/mod.rs
index aa6f207ead1a9..f4f9e931fa1a0 100644
--- a/src/common/src/system_param/mod.rs
+++ b/src/common/src/system_param/mod.rs
@@ -99,6 +99,7 @@ macro_rules! for_all_params {
             { enable_tracing,                           bool,   Some(false),                                true,   "Whether to enable distributed tracing.", },
             { use_new_object_prefix_strategy,           bool,   None,                                       false,  "Whether to split object prefix.", },
             { license_key,                              String, Some(default_license_key()),                true,   "The license key to activate enterprise features.", },
+            { time_travel_retention_ms,                 u64,    Some(0_u64),                                true,   "The data retention period for time travel, where 0 indicates that it's disabled.", },
         }
     };
 }
@@ -458,6 +459,7 @@ mod tests {
             (ENABLE_TRACING_KEY, "true"),
             (USE_NEW_OBJECT_PREFIX_STRATEGY_KEY, "false"),
             (LICENSE_KEY_KEY, "foo"),
+            (TIME_TRAVEL_RETENTION_MS_KEY, "0"),
             ("a_deprecated_param", "foo"),
         ];
 
diff --git a/src/common/src/system_param/reader.rs b/src/common/src/system_param/reader.rs
index 2ef13ff3f5509..505e36693f1d5 100644
--- a/src/common/src/system_param/reader.rs
+++ b/src/common/src/system_param/reader.rs
@@ -172,4 +172,10 @@ where
     fn license_key(&self) -> &str {
         self.inner().license_key.as_deref().unwrap_or_default()
     }
+
+    fn time_travel_retention_ms(&self) -> u64 {
+        self.inner()
+            .time_travel_retention_ms
+            .unwrap_or_else(default::time_travel_retention_ms)
+    }
 }
diff --git a/src/common/src/telemetry/report.rs b/src/common/src/telemetry/report.rs
index 80c5cfa14d114..38d1c48065635 100644
--- a/src/common/src/telemetry/report.rs
+++ b/src/common/src/telemetry/report.rs
@@ -12,14 +12,19 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::sync::Arc;
+use std::sync::{Arc, OnceLock};
 
+use prost::Message;
+use risingwave_pb::telemetry::{
+    EventMessage as PbEventMessage, PbTelemetryDatabaseObject,
+    TelemetryEventStage as PbTelemetryEventStage,
+};
 use tokio::sync::oneshot::Sender;
 use tokio::task::JoinHandle;
 use tokio::time::{interval, Duration};
 use uuid::Uuid;
 
-use super::{Result, TELEMETRY_REPORT_INTERVAL, TELEMETRY_REPORT_URL};
+use super::{current_timestamp, Result, TELEMETRY_REPORT_INTERVAL, TELEMETRY_REPORT_URL};
 use crate::telemetry::pb_compatible::TelemetryToProtobuf;
 use crate::telemetry::post_telemetry_report_pb;
 
@@ -42,6 +47,8 @@ pub trait TelemetryReportCreator {
     fn report_type(&self) -> &str;
 }
 
+static TELEMETRY_TRACKING_ID: OnceLock<String> = OnceLock::new();
+
 pub async fn start_telemetry_reporting<F, I>(
     info_fetcher: Arc<I>,
     report_creator: Arc<F>,
@@ -74,6 +81,13 @@ where
                 return;
             }
         };
+        TELEMETRY_TRACKING_ID
+            .set(tracking_id.clone())
+            .unwrap_or_else(|_| {
+                tracing::warn!(
+                    "Telemetry failed to set tracking_id, event reporting will be disabled"
+                )
+            });
 
         loop {
             tokio::select! {
@@ -112,3 +126,99 @@ where
     });
     (join_handle, shutdown_tx)
 }
+
+pub fn report_event_common(
+    event_stage: PbTelemetryEventStage,
+    event_name: &str,
+    catalog_id: i64,
+    connector_name: Option<String>,
+    object: Option<PbTelemetryDatabaseObject>,
+    attributes: Option<jsonbb::Value>, // any json string
+    node: String,
+) {
+    let event_tracking_id: String;
+    if let Some(tracking_id) = TELEMETRY_TRACKING_ID.get() {
+        event_tracking_id = tracking_id.to_string();
+    } else {
+        tracing::info!("Telemetry tracking_id is not set, event reporting disabled");
+        return;
+    }
+
+    request_to_telemetry_event(
+        event_tracking_id,
+        event_stage,
+        event_name,
+        catalog_id,
+        connector_name,
+        object,
+        attributes,
+        node,
+        false,
+    );
+}
+
+fn request_to_telemetry_event(
+    tracking_id: String,
+    event_stage: PbTelemetryEventStage,
+    event_name: &str,
+    catalog_id: i64,
+    connector_name: Option<String>,
+    object: Option<PbTelemetryDatabaseObject>,
+    attributes: Option<jsonbb::Value>, // any json string
+    node: String,
+    is_test: bool,
+) {
+    let event = PbEventMessage {
+        tracking_id,
+        event_time_sec: current_timestamp(),
+        event_stage: event_stage as i32,
+        event_name: event_name.to_string(),
+        connector_name,
+        object: object.map(|c| c as i32),
+        catalog_id,
+        attributes: attributes.map(|a| a.to_string()),
+        node,
+        is_test,
+    };
+    let report_bytes = event.encode_to_vec();
+
+    tokio::spawn(async move {
+        const TELEMETRY_EVENT_REPORT_TYPE: &str = "event";
+        let url = (TELEMETRY_REPORT_URL.to_owned() + "/" + TELEMETRY_EVENT_REPORT_TYPE).to_owned();
+        post_telemetry_report_pb(&url, report_bytes)
+            .await
+            .unwrap_or_else(|e| tracing::info!("{}", e))
+    });
+}
+
+#[cfg(test)]
+mod test {
+
+    use super::*;
+
+    #[ignore]
+    #[tokio::test]
+    async fn test_telemetry_report_event() {
+        let event_stage = PbTelemetryEventStage::CreateStreamJob;
+        let event_name = "test_feature";
+        let catalog_id = 1;
+        let connector_name = Some("test_connector".to_string());
+        let object = Some(PbTelemetryDatabaseObject::Source);
+        let attributes = None;
+        let node = "test_node".to_string();
+
+        request_to_telemetry_event(
+            "7d45669c-08c7-4571-ae3d-d3a3e70a2f7e".to_string(),
+            event_stage,
+            event_name,
+            catalog_id,
+            connector_name,
+            object,
+            attributes,
+            node,
+            true,
+        );
+
+        tokio::time::sleep(tokio::time::Duration::from_secs(1)).await;
+    }
+}
diff --git a/src/common/src/test_utils/rand_array.rs b/src/common/src/test_utils/rand_array.rs
index b7c1d3630b9b7..0e2e1d92e28c3 100644
--- a/src/common/src/test_utils/rand_array.rs
+++ b/src/common/src/test_utils/rand_array.rs
@@ -86,7 +86,7 @@ impl RandValue for Date {
         let max_day = chrono::NaiveDate::MAX.num_days_from_ce();
         let min_day = chrono::NaiveDate::MIN.num_days_from_ce();
         let days = rand.gen_range(min_day..=max_day);
-        Date::with_days(days).unwrap()
+        Date::with_days_since_ce(days).unwrap()
     }
 }
 
diff --git a/src/common/src/test_utils/rand_bitmap.rs b/src/common/src/test_utils/rand_bitmap.rs
index 8e76381b54556..a6e8c325dc513 100644
--- a/src/common/src/test_utils/rand_bitmap.rs
+++ b/src/common/src/test_utils/rand_bitmap.rs
@@ -16,7 +16,7 @@ use itertools::Itertools;
 use rand::seq::SliceRandom;
 use rand::SeedableRng;
 
-use crate::buffer::{Bitmap, BitmapBuilder};
+use crate::bitmap::{Bitmap, BitmapBuilder};
 
 pub fn gen_rand_bitmap(num_bits: usize, count_ones: usize, seed: u64) -> Bitmap {
     if count_ones == num_bits {
diff --git a/src/common/src/test_utils/rand_stream_chunk.rs b/src/common/src/test_utils/rand_stream_chunk.rs
index 34cbe5627a54b..6c066bfbbd7ca 100644
--- a/src/common/src/test_utils/rand_stream_chunk.rs
+++ b/src/common/src/test_utils/rand_stream_chunk.rs
@@ -16,7 +16,7 @@ use rand::{Rng, SeedableRng};
 
 use crate::array::stream_chunk::Op;
 use crate::array::{ArrayBuilder, ArrayImpl, I64ArrayBuilder};
-use crate::buffer::Bitmap;
+use crate::bitmap::Bitmap;
 
 pub fn gen_legal_stream_chunk(
     bitmap: &Bitmap,
diff --git a/src/common/src/types/datetime.rs b/src/common/src/types/datetime.rs
index bac96b6c1dea5..1b8d004c68b2b 100644
--- a/src/common/src/types/datetime.rs
+++ b/src/common/src/types/datetime.rs
@@ -17,10 +17,12 @@
 use std::error::Error;
 use std::fmt::Display;
 use std::hash::Hash;
-use std::io::Write;
+use std::io::{Cursor, Write};
 use std::str::FromStr;
 
-use bytes::{Bytes, BytesMut};
+use anyhow::Context;
+use byteorder::{BigEndian, ReadBytesExt};
+use bytes::BytesMut;
 use chrono::{
     DateTime, Datelike, Days, Duration, NaiveDate, NaiveDateTime, NaiveTime, Timelike, Weekday,
 };
@@ -28,7 +30,6 @@ use postgres_types::{accepts, to_sql_checked, FromSql, IsNull, ToSql, Type};
 use risingwave_common_estimate_size::ZeroHeapSize;
 use thiserror::Error;
 
-use super::to_binary::ToBinary;
 use super::to_text::ToText;
 use super::{CheckedAdd, DataType, Interval};
 use crate::array::{ArrayError, ArrayResult};
@@ -40,7 +41,7 @@ const LEAP_DAYS: &[i32] = &[0, 31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31];
 const NORMAL_DAYS: &[i32] = &[0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31];
 
 macro_rules! impl_chrono_wrapper {
-    ($variant_name:ident, $chrono:ty) => {
+    ($variant_name:ident, $chrono:ty, $pg_type:ident) => {
         #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash)]
         #[repr(transparent)]
         pub struct $variant_name(pub $chrono);
@@ -66,105 +67,43 @@ macro_rules! impl_chrono_wrapper {
         }
 
         impl ZeroHeapSize for $variant_name {}
-    };
-}
-
-impl_chrono_wrapper!(Date, NaiveDate);
-impl_chrono_wrapper!(Timestamp, NaiveDateTime);
-impl_chrono_wrapper!(Time, NaiveTime);
-
-impl ToSql for Date {
-    accepts!(DATE);
-
-    to_sql_checked!();
-
-    fn to_sql(
-        &self,
-        ty: &Type,
-        out: &mut BytesMut,
-    ) -> std::result::Result<IsNull, Box<dyn Error + Sync + Send>>
-    where
-        Self: Sized,
-    {
-        self.0.to_sql(ty, out)
-    }
-}
 
-impl<'a> FromSql<'a> for Date {
-    fn from_sql(
-        ty: &Type,
-        raw: &'a [u8],
-    ) -> std::result::Result<Self, Box<dyn std::error::Error + Sync + Send>> {
-        let instant = NaiveDate::from_sql(ty, raw)?;
-        Ok(Self::from(instant))
-    }
-
-    fn accepts(ty: &Type) -> bool {
-        matches!(*ty, Type::DATE)
-    }
-}
+        impl ToSql for $variant_name {
+            accepts!($pg_type);
 
-impl ToSql for Time {
-    accepts!(TIME);
-
-    to_sql_checked!();
-
-    fn to_sql(
-        &self,
-        ty: &Type,
-        out: &mut BytesMut,
-    ) -> std::result::Result<IsNull, Box<dyn Error + Sync + Send>>
-    where
-        Self: Sized,
-    {
-        self.0.to_sql(ty, out)
-    }
-}
+            to_sql_checked!();
 
-impl<'a> FromSql<'a> for Time {
-    fn from_sql(
-        ty: &Type,
-        raw: &'a [u8],
-    ) -> std::result::Result<Self, Box<dyn std::error::Error + Sync + Send>> {
-        let instant = NaiveTime::from_sql(ty, raw)?;
-        Ok(Self::from(instant))
-    }
-
-    fn accepts(ty: &Type) -> bool {
-        matches!(*ty, Type::TIME)
-    }
-}
-
-impl ToSql for Timestamp {
-    accepts!(TIMESTAMP);
+            fn to_sql(
+                &self,
+                ty: &Type,
+                out: &mut BytesMut,
+            ) -> std::result::Result<IsNull, Box<dyn Error + Sync + Send>>
+            where
+                Self: Sized,
+            {
+                self.0.to_sql(ty, out)
+            }
+        }
 
-    to_sql_checked!();
+        impl<'a> FromSql<'a> for $variant_name {
+            fn from_sql(
+                ty: &Type,
+                raw: &'a [u8],
+            ) -> std::result::Result<Self, Box<dyn std::error::Error + Sync + Send>> {
+                let instant = <$chrono>::from_sql(ty, raw)?;
+                Ok(Self::from(instant))
+            }
 
-    fn to_sql(
-        &self,
-        ty: &Type,
-        out: &mut BytesMut,
-    ) -> std::result::Result<IsNull, Box<dyn Error + Sync + Send>>
-    where
-        Self: Sized,
-    {
-        self.0.to_sql(ty, out)
-    }
+            fn accepts(ty: &Type) -> bool {
+                matches!(*ty, Type::$pg_type)
+            }
+        }
+    };
 }
 
-impl<'a> FromSql<'a> for Timestamp {
-    fn from_sql(
-        ty: &Type,
-        raw: &'a [u8],
-    ) -> std::result::Result<Self, Box<dyn std::error::Error + Sync + Send>> {
-        let instant = NaiveDateTime::from_sql(ty, raw)?;
-        Ok(Self::from(instant))
-    }
-
-    fn accepts(ty: &Type) -> bool {
-        matches!(*ty, Type::TIMESTAMP)
-    }
-}
+impl_chrono_wrapper!(Date, NaiveDate, DATE);
+impl_chrono_wrapper!(Timestamp, NaiveDateTime, TIMESTAMP);
+impl_chrono_wrapper!(Time, NaiveTime, TIME);
 
 /// Parse a date from varchar.
 ///
@@ -427,47 +366,8 @@ impl ToText for Timestamp {
     }
 }
 
-impl ToBinary for Date {
-    fn to_binary_with_type(&self, ty: &DataType) -> super::to_binary::Result<Option<Bytes>> {
-        match ty {
-            super::DataType::Date => {
-                let mut output = BytesMut::new();
-                self.0.to_sql(&Type::ANY, &mut output).unwrap();
-                Ok(Some(output.freeze()))
-            }
-            _ => unreachable!(),
-        }
-    }
-}
-
-impl ToBinary for Time {
-    fn to_binary_with_type(&self, ty: &DataType) -> super::to_binary::Result<Option<Bytes>> {
-        match ty {
-            super::DataType::Time => {
-                let mut output = BytesMut::new();
-                self.0.to_sql(&Type::ANY, &mut output).unwrap();
-                Ok(Some(output.freeze()))
-            }
-            _ => unreachable!(),
-        }
-    }
-}
-
-impl ToBinary for Timestamp {
-    fn to_binary_with_type(&self, ty: &DataType) -> super::to_binary::Result<Option<Bytes>> {
-        match ty {
-            super::DataType::Timestamp => {
-                let mut output = BytesMut::new();
-                self.0.to_sql(&Type::ANY, &mut output).unwrap();
-                Ok(Some(output.freeze()))
-            }
-            _ => unreachable!(),
-        }
-    }
-}
-
 impl Date {
-    pub fn with_days(days: i32) -> Result<Self> {
+    pub fn with_days_since_ce(days: i32) -> Result<Self> {
         Ok(Date::new(
             NaiveDate::from_num_days_from_ce_opt(days)
                 .ok_or_else(|| InvalidParamsError::date(days))?,
@@ -490,6 +390,14 @@ impl Date {
             .num_days_from_ce()
     }
 
+    pub fn from_protobuf(cur: &mut Cursor<&[u8]>) -> ArrayResult<Date> {
+        let days = cur
+            .read_i32::<BigEndian>()
+            .context("failed to read i32 from Date buffer")?;
+
+        Ok(Date::with_days_since_ce(days)?)
+    }
+
     pub fn to_protobuf<T: Write>(self, output: &mut T) -> ArrayResult<usize> {
         output
             .write(&(self.0.num_days_from_ce()).to_be_bytes())
@@ -501,7 +409,7 @@ impl Date {
     }
 
     pub fn from_num_days_from_ce_uncheck(days: i32) -> Self {
-        Self::with_days(days).unwrap()
+        Self::with_days_since_ce(days).unwrap()
     }
 
     pub fn and_hms_uncheck(self, hour: u32, min: u32, sec: u32) -> Timestamp {
@@ -524,6 +432,14 @@ impl Time {
         ))
     }
 
+    pub fn from_protobuf(cur: &mut Cursor<&[u8]>) -> ArrayResult<Time> {
+        let nano = cur
+            .read_u64::<BigEndian>()
+            .context("failed to read u64 from Time buffer")?;
+
+        Ok(Time::with_nano(nano)?)
+    }
+
     pub fn to_protobuf<T: Write>(self, output: &mut T) -> ArrayResult<usize> {
         output
             .write(
@@ -578,6 +494,14 @@ impl Timestamp {
         }))
     }
 
+    pub fn from_protobuf(cur: &mut Cursor<&[u8]>) -> ArrayResult<Timestamp> {
+        let micros = cur
+            .read_i64::<BigEndian>()
+            .context("failed to read i64 from Timestamp buffer")?;
+
+        Ok(Timestamp::with_micros(micros)?)
+    }
+
     /// Although `Timestamp` takes 12 bytes, we drop 4 bytes in protobuf encoding.
     pub fn to_protobuf<T: Write>(self, output: &mut T) -> ArrayResult<usize> {
         output
diff --git a/src/common/src/types/decimal.rs b/src/common/src/types/decimal.rs
index b38dbb4822e68..9523157239e2e 100644
--- a/src/common/src/types/decimal.rs
+++ b/src/common/src/types/decimal.rs
@@ -17,7 +17,7 @@ use std::io::{Cursor, Read, Write};
 use std::ops::{Add, Div, Mul, Neg, Rem, Sub};
 
 use byteorder::{BigEndian, ReadBytesExt};
-use bytes::{BufMut, Bytes, BytesMut};
+use bytes::{BufMut, BytesMut};
 use num_traits::{
     CheckedAdd, CheckedDiv, CheckedMul, CheckedNeg, CheckedRem, CheckedSub, Num, One, Zero,
 };
@@ -26,7 +26,6 @@ use risingwave_common_estimate_size::ZeroHeapSize;
 use rust_decimal::prelude::FromStr;
 use rust_decimal::{Decimal as RustDecimal, Error, MathematicalOps as _, RoundingStrategy};
 
-use super::to_binary::ToBinary;
 use super::to_text::ToText;
 use super::DataType;
 use crate::array::ArrayResult;
@@ -90,19 +89,6 @@ impl Decimal {
     }
 }
 
-impl ToBinary for Decimal {
-    fn to_binary_with_type(&self, ty: &DataType) -> super::to_binary::Result<Option<Bytes>> {
-        match ty {
-            DataType::Decimal => {
-                let mut output = BytesMut::new();
-                self.to_sql(&Type::NUMERIC, &mut output).unwrap();
-                Ok(Some(output.freeze()))
-            }
-            _ => unreachable!(),
-        }
-    }
-}
-
 impl ToSql for Decimal {
     accepts!(NUMERIC);
 
diff --git a/src/common/src/types/interval.rs b/src/common/src/types/interval.rs
index 495561561e937..30845805ede79 100644
--- a/src/common/src/types/interval.rs
+++ b/src/common/src/types/interval.rs
@@ -16,10 +16,11 @@ use std::cmp::Ordering;
 use std::error::Error;
 use std::fmt::{Display, Formatter};
 use std::hash::{Hash, Hasher};
-use std::io::Write;
+use std::io::{Cursor, Write};
 use std::ops::{Add, Neg, Sub};
 use std::sync::LazyLock;
 
+use anyhow::Context;
 use byteorder::{BigEndian, NetworkEndian, ReadBytesExt, WriteBytesExt};
 use bytes::BytesMut;
 use num_traits::{CheckedAdd, CheckedNeg, CheckedSub, Zero};
@@ -229,6 +230,18 @@ impl Interval {
         secs_from_day_month as i128 * USECS_PER_SEC as i128 + self.usecs as i128
     }
 
+    pub fn from_protobuf(cursor: &mut Cursor<&[u8]>) -> ArrayResult<Interval> {
+        let mut read = || {
+            let months = cursor.read_i32::<BigEndian>()?;
+            let days = cursor.read_i32::<BigEndian>()?;
+            let usecs = cursor.read_i64::<BigEndian>()?;
+
+            Ok::<_, std::io::Error>(Interval::from_month_day_usec(months, days, usecs))
+        };
+
+        Ok(read().context("failed to read Interval from buffer")?)
+    }
+
     pub fn to_protobuf<T: Write>(self, output: &mut T) -> ArrayResult<usize> {
         output.write_i32::<BigEndian>(self.months)?;
         output.write_i32::<BigEndian>(self.days)?;
@@ -1181,19 +1194,6 @@ impl<'a> FromSql<'a> for Interval {
     }
 }
 
-impl ToBinary for Interval {
-    fn to_binary_with_type(&self, ty: &DataType) -> super::to_binary::Result<Option<Bytes>> {
-        match ty {
-            DataType::Interval => {
-                let mut output = BytesMut::new();
-                self.to_sql(&Type::ANY, &mut output).unwrap();
-                Ok(Some(output.freeze()))
-            }
-            _ => unreachable!(),
-        }
-    }
-}
-
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
 pub enum DateTimeField {
     Year,
diff --git a/src/common/src/types/jsonb.rs b/src/common/src/types/jsonb.rs
index 642b363a8c67e..fa80069080ff4 100644
--- a/src/common/src/types/jsonb.rs
+++ b/src/common/src/types/jsonb.rs
@@ -301,6 +301,14 @@ impl<'a> JsonbRef<'a> {
             .ok_or_else(|| format!("cannot cast jsonb {} to type boolean", self.type_name()))
     }
 
+    /// If the JSON is a string, returns the associated string.
+    pub fn as_string(&self) -> Result<String, String> {
+        self.0
+            .as_str()
+            .map(|s| s.to_owned())
+            .ok_or_else(|| format!("cannot cast jsonb {} to type string", self.type_name()))
+    }
+
     /// Attempt to read jsonb as a JSON number.
     ///
     /// According to RFC 8259, only number within IEEE 754 binary64 (double precision) has good
diff --git a/src/common/src/types/macros.rs b/src/common/src/types/macros.rs
index 35f106aafdffd..520e4ab8f45ee 100644
--- a/src/common/src/types/macros.rs
+++ b/src/common/src/types/macros.rs
@@ -39,6 +39,7 @@ macro_rules! for_all_variants {
     ($macro:ident $(, $x:tt)*) => {
         $macro! {
             $($x, )*
+            //data_type     variant_name  suffix_name   scalar                      scalar_ref                          array                               builder
             { Int16,        Int16,        int16,        i16,                        i16,                                $crate::array::I16Array,            $crate::array::I16ArrayBuilder          },
             { Int32,        Int32,        int32,        i32,                        i32,                                $crate::array::I32Array,            $crate::array::I32ArrayBuilder          },
             { Int64,        Int64,        int64,        i64,                        i64,                                $crate::array::I64Array,            $crate::array::I64ArrayBuilder          },
diff --git a/src/common/src/types/mod.rs b/src/common/src/types/mod.rs
index 69e727cbde655..e53a4597301b0 100644
--- a/src/common/src/types/mod.rs
+++ b/src/common/src/types/mod.rs
@@ -89,7 +89,6 @@ pub use self::serial::Serial;
 pub use self::struct_type::StructType;
 pub use self::successor::Successor;
 pub use self::timestamptz::*;
-pub use self::to_binary::ToBinary;
 pub use self::to_text::ToText;
 pub use self::with_data_type::WithDataType;
 
@@ -179,67 +178,39 @@ impl std::str::FromStr for Box<DataType> {
 
 impl ZeroHeapSize for DataType {}
 
-impl DataTypeName {
-    pub fn is_scalar(&self) -> bool {
-        match self {
-            DataTypeName::Boolean
-            | DataTypeName::Int16
-            | DataTypeName::Int32
-            | DataTypeName::Int64
-            | DataTypeName::Int256
-            | DataTypeName::Serial
-            | DataTypeName::Decimal
-            | DataTypeName::Float32
-            | DataTypeName::Float64
-            | DataTypeName::Varchar
-            | DataTypeName::Date
-            | DataTypeName::Timestamp
-            | DataTypeName::Timestamptz
-            | DataTypeName::Time
-            | DataTypeName::Bytea
-            | DataTypeName::Jsonb
-            | DataTypeName::Interval => true,
-
-            DataTypeName::Struct | DataTypeName::List => false,
-        }
-    }
+impl TryFrom<DataTypeName> for DataType {
+    type Error = &'static str;
 
-    pub fn to_type(self) -> Option<DataType> {
-        let t = match self {
-            DataTypeName::Boolean => DataType::Boolean,
-            DataTypeName::Int16 => DataType::Int16,
-            DataTypeName::Int32 => DataType::Int32,
-            DataTypeName::Int64 => DataType::Int64,
-            DataTypeName::Int256 => DataType::Int256,
-            DataTypeName::Serial => DataType::Serial,
-            DataTypeName::Decimal => DataType::Decimal,
-            DataTypeName::Float32 => DataType::Float32,
-            DataTypeName::Float64 => DataType::Float64,
-            DataTypeName::Varchar => DataType::Varchar,
-            DataTypeName::Bytea => DataType::Bytea,
-            DataTypeName::Date => DataType::Date,
-            DataTypeName::Timestamp => DataType::Timestamp,
-            DataTypeName::Timestamptz => DataType::Timestamptz,
-            DataTypeName::Time => DataType::Time,
-            DataTypeName::Interval => DataType::Interval,
-            DataTypeName::Jsonb => DataType::Jsonb,
+    fn try_from(type_name: DataTypeName) -> Result<Self, Self::Error> {
+        match type_name {
+            DataTypeName::Boolean => Ok(DataType::Boolean),
+            DataTypeName::Int16 => Ok(DataType::Int16),
+            DataTypeName::Int32 => Ok(DataType::Int32),
+            DataTypeName::Int64 => Ok(DataType::Int64),
+            DataTypeName::Int256 => Ok(DataType::Int256),
+            DataTypeName::Serial => Ok(DataType::Serial),
+            DataTypeName::Decimal => Ok(DataType::Decimal),
+            DataTypeName::Float32 => Ok(DataType::Float32),
+            DataTypeName::Float64 => Ok(DataType::Float64),
+            DataTypeName::Varchar => Ok(DataType::Varchar),
+            DataTypeName::Bytea => Ok(DataType::Bytea),
+            DataTypeName::Date => Ok(DataType::Date),
+            DataTypeName::Timestamp => Ok(DataType::Timestamp),
+            DataTypeName::Timestamptz => Ok(DataType::Timestamptz),
+            DataTypeName::Time => Ok(DataType::Time),
+            DataTypeName::Interval => Ok(DataType::Interval),
+            DataTypeName::Jsonb => Ok(DataType::Jsonb),
             DataTypeName::Struct | DataTypeName::List => {
-                return None;
+                Err("Functions returning struct or list can not be inferred. Please use `FunctionCall::new_unchecked`.")
             }
-        };
-        Some(t)
-    }
-}
-
-impl From<DataTypeName> for DataType {
-    fn from(type_name: DataTypeName) -> Self {
-        type_name.to_type().unwrap_or_else(|| panic!("Functions returning struct or list can not be inferred. Please use `FunctionCall::new_unchecked`."))
+        }
     }
 }
 
 impl From<&PbDataType> for DataType {
     fn from(proto: &PbDataType) -> DataType {
         match proto.get_type_name().expect("missing type field") {
+            PbTypeName::TypeUnspecified => unreachable!(),
             PbTypeName::Int16 => DataType::Int16,
             PbTypeName::Int32 => DataType::Int32,
             PbTypeName::Int64 => DataType::Int64,
@@ -265,7 +236,6 @@ impl From<&PbDataType> for DataType {
                 // The first (and only) item is the list element type.
                 Box::new((&proto.field_type[0]).into()),
             ),
-            PbTypeName::TypeUnspecified => unreachable!(),
             PbTypeName::Int256 => DataType::Int256,
         }
     }
@@ -337,27 +307,7 @@ impl DataType {
     }
 
     pub fn prost_type_name(&self) -> PbTypeName {
-        match self {
-            DataType::Int16 => PbTypeName::Int16,
-            DataType::Int32 => PbTypeName::Int32,
-            DataType::Int64 => PbTypeName::Int64,
-            DataType::Int256 => PbTypeName::Int256,
-            DataType::Serial => PbTypeName::Serial,
-            DataType::Float32 => PbTypeName::Float,
-            DataType::Float64 => PbTypeName::Double,
-            DataType::Boolean => PbTypeName::Boolean,
-            DataType::Varchar => PbTypeName::Varchar,
-            DataType::Date => PbTypeName::Date,
-            DataType::Time => PbTypeName::Time,
-            DataType::Timestamp => PbTypeName::Timestamp,
-            DataType::Timestamptz => PbTypeName::Timestamptz,
-            DataType::Decimal => PbTypeName::Decimal,
-            DataType::Interval => PbTypeName::Interval,
-            DataType::Jsonb => PbTypeName::Jsonb,
-            DataType::Struct { .. } => PbTypeName::Struct,
-            DataType::List { .. } => PbTypeName::List,
-            DataType::Bytea => PbTypeName::Bytea,
-        }
+        DataTypeName::from(self).into()
     }
 
     pub fn to_protobuf(&self) -> PbDataType {
@@ -374,7 +324,23 @@ impl DataType {
             DataType::List(datatype) => {
                 pb.field_type = vec![datatype.to_protobuf()];
             }
-            _ => {}
+            DataType::Boolean
+            | DataType::Int16
+            | DataType::Int32
+            | DataType::Int64
+            | DataType::Float32
+            | DataType::Float64
+            | DataType::Decimal
+            | DataType::Date
+            | DataType::Varchar
+            | DataType::Time
+            | DataType::Timestamp
+            | DataType::Timestamptz
+            | DataType::Interval
+            | DataType::Bytea
+            | DataType::Jsonb
+            | DataType::Serial
+            | DataType::Int256 => (),
         }
         pb
     }
@@ -392,10 +358,6 @@ impl DataType {
         )
     }
 
-    pub fn is_scalar(&self) -> bool {
-        DataTypeName::from(self).is_scalar()
-    }
-
     pub fn is_array(&self) -> bool {
         matches!(self, DataType::List(_))
     }
@@ -440,37 +402,6 @@ impl DataType {
         }
     }
 
-    /// WARNING: Currently this should only be used in `WatermarkFilterExecutor`. Please be careful
-    /// if you want to use this.
-    pub fn min_value(&self) -> ScalarImpl {
-        match self {
-            DataType::Int16 => ScalarImpl::Int16(i16::MIN),
-            DataType::Int32 => ScalarImpl::Int32(i32::MIN),
-            DataType::Int64 => ScalarImpl::Int64(i64::MIN),
-            DataType::Int256 => ScalarImpl::Int256(Int256::min_value()),
-            DataType::Serial => ScalarImpl::Serial(Serial::from(i64::MIN)),
-            DataType::Float32 => ScalarImpl::Float32(F32::neg_infinity()),
-            DataType::Float64 => ScalarImpl::Float64(F64::neg_infinity()),
-            DataType::Boolean => ScalarImpl::Bool(false),
-            DataType::Varchar => ScalarImpl::Utf8("".into()),
-            DataType::Bytea => ScalarImpl::Bytea("".to_string().into_bytes().into()),
-            DataType::Date => ScalarImpl::Date(Date::MIN),
-            DataType::Time => ScalarImpl::Time(Time::from_hms_uncheck(0, 0, 0)),
-            DataType::Timestamp => ScalarImpl::Timestamp(Timestamp::MIN),
-            DataType::Timestamptz => ScalarImpl::Timestamptz(Timestamptz::MIN),
-            DataType::Decimal => ScalarImpl::Decimal(Decimal::NegativeInf),
-            DataType::Interval => ScalarImpl::Interval(Interval::MIN),
-            DataType::Jsonb => ScalarImpl::Jsonb(JsonbVal::null()), // NOT `min` #7981
-            DataType::Struct(data_types) => ScalarImpl::Struct(StructValue::new(
-                data_types
-                    .types()
-                    .map(|data_type| Some(data_type.min_value()))
-                    .collect_vec(),
-            )),
-            DataType::List(data_type) => ScalarImpl::List(ListValue::empty(data_type)),
-        }
-    }
-
     /// Return a new type that removes the outer list.
     ///
     /// ```
@@ -513,28 +444,32 @@ impl From<DataType> for PbDataType {
     }
 }
 
-/// Common trait bounds of scalar and scalar reference types.
-///
-/// NOTE(rc): `Hash` is not in the trait bound list, it's implemented as [`ScalarRef::hash_scalar`].
-pub trait ScalarBounds<Impl> = Debug
-    + Send
-    + Sync
-    + Clone
-    + PartialEq
-    + Eq
-    // in default ascending order
-    + PartialOrd
-    + Ord
-    + TryFrom<Impl, Error = ArrayError>
-    // `ScalarImpl`/`ScalarRefImpl`
-    + Into<Impl>;
+mod private {
+    use super::*;
+
+    /// Common trait bounds of scalar and scalar reference types.
+    ///
+    /// NOTE(rc): `Hash` is not in the trait bound list, it's implemented as [`ScalarRef::hash_scalar`].
+    pub trait ScalarBounds<Impl> = Debug
+        + Send
+        + Sync
+        + Clone
+        + PartialEq
+        + Eq
+        // in default ascending order
+        + PartialOrd
+        + Ord
+        + TryFrom<Impl, Error = ArrayError>
+        // `ScalarImpl`/`ScalarRefImpl`
+        + Into<Impl>;
+}
 
 /// `Scalar` is a trait over all possible owned types in the evaluation
 /// framework.
 ///
 /// `Scalar` is reciprocal to `ScalarRef`. Use `as_scalar_ref` to get a
 /// reference which has the same lifetime as `self`.
-pub trait Scalar: ScalarBounds<ScalarImpl> + 'static {
+pub trait Scalar: private::ScalarBounds<ScalarImpl> + 'static {
     /// Type for reference of `Scalar`
     type ScalarRefType<'a>: ScalarRef<'a, ScalarType = Self> + 'a
     where
@@ -548,17 +483,12 @@ pub trait Scalar: ScalarBounds<ScalarImpl> + 'static {
     }
 }
 
-/// Convert an `Option<Scalar>` to corresponding `Option<ScalarRef>`.
-pub fn option_as_scalar_ref<S: Scalar>(scalar: &Option<S>) -> Option<S::ScalarRefType<'_>> {
-    scalar.as_ref().map(|x| x.as_scalar_ref())
-}
-
 /// `ScalarRef` is a trait over all possible references in the evaluation
 /// framework.
 ///
 /// `ScalarRef` is reciprocal to `Scalar`. Use `to_owned_scalar` to get an
 /// owned scalar.
-pub trait ScalarRef<'a>: ScalarBounds<ScalarRefImpl<'a>> + 'a + Copy {
+pub trait ScalarRef<'a>: private::ScalarBounds<ScalarRefImpl<'a>> + 'a + Copy {
     /// `ScalarType` is the owned type of current `ScalarRef`.
     type ScalarType: Scalar<ScalarRefType<'a> = Self>;
 
@@ -573,6 +503,12 @@ pub trait ScalarRef<'a>: ScalarBounds<ScalarRefImpl<'a>> + 'a + Copy {
 macro_rules! scalar_impl_enum {
     ($( { $variant_name:ident, $suffix_name:ident, $scalar:ty, $scalar_ref:ty } ),*) => {
         /// `ScalarImpl` embeds all possible scalars in the evaluation framework.
+        ///
+        /// Note: `ScalarImpl` doesn't contain all information of its `DataType`,
+        /// so sometimes they need to be used together.
+        /// e.g., for `Struct`, we don't have the field names in the value.
+        ///
+        /// See `for_all_variants` for the definition.
         #[derive(Debug, Clone, PartialEq, Eq, EstimateSize)]
         pub enum ScalarImpl {
             $( $variant_name($scalar) ),*
@@ -580,6 +516,12 @@ macro_rules! scalar_impl_enum {
 
         /// `ScalarRefImpl` embeds all possible scalar references in the evaluation
         /// framework.
+        ///
+        /// Note: `ScalarRefImpl` doesn't contain all information of its `DataType`,
+        /// so sometimes they need to be used together.
+        /// e.g., for `Struct`, we don't have the field names in the value.
+        ///
+        /// See `for_all_variants` for the definition.
         #[derive(Debug, Copy, Clone, PartialEq, Eq)]
         pub enum ScalarRefImpl<'scalar> {
             $( $variant_name($scalar_ref) ),*
@@ -651,6 +593,9 @@ impl ToDatumRef for DatumRef<'_> {
 }
 
 /// To make sure there is `as_scalar_ref` for all scalar ref types.
+/// See <https://github.com/risingwavelabs/risingwave/pull/9977/files#r1208972881>
+///
+/// This is used by the expr macro.
 pub trait SelfAsScalarRef {
     fn as_scalar_ref(&self) -> Self;
 }
@@ -674,12 +619,12 @@ impl_self_as_scalar_ref! { &str, &[u8], Int256Ref<'_>, JsonbRef<'_>, ListRef<'_>
 macro_rules! for_all_native_types {
     ($macro:ident) => {
         $macro! {
-            { i16, Int16 },
-            { i32, Int32 },
-            { i64, Int64 },
-            { Serial, Serial },
-            { $crate::types::F32, Float32 },
-            { $crate::types::F64, Float64 }
+            { i16, Int16, read_i16 },
+            { i32, Int32, read_i32 },
+            { i64, Int64, read_i64 },
+            { Serial, Serial, read_i64 },
+            { $crate::types::F32, Float32, read_f32 },
+            { $crate::types::F64, Float64, read_f64 }
         }
     };
 }
@@ -855,7 +800,9 @@ impl From<Bytes> for ScalarImpl {
 }
 
 impl ScalarImpl {
-    /// Creates a scalar from binary.
+    /// Creates a scalar from pgwire "BINARY" format.
+    ///
+    /// The counterpart of [`to_binary::ToBinary`].
     pub fn from_binary(bytes: &Bytes, data_type: &DataType) -> Result<Self, BoxedError> {
         let res = match data_type {
             DataType::Varchar => Self::Utf8(String::from_sql(&Type::VARCHAR, bytes)?.into()),
@@ -891,7 +838,9 @@ impl ScalarImpl {
         Ok(res)
     }
 
-    /// Creates a scalar from text.
+    /// Creates a scalar from pgwire "TEXT" format.
+    ///
+    /// The counterpart of [`ToText`].
     pub fn from_text(s: &str, data_type: &DataType) -> Result<Self, BoxedError> {
         Ok(match data_type {
             DataType::Boolean => str_to_bool(s)?.into(),
@@ -972,9 +921,8 @@ pub fn hash_datum(datum: impl ToDatumRef, state: &mut impl std::hash::Hasher) {
 }
 
 impl ScalarRefImpl<'_> {
-    /// Encode the scalar to postgresql binary format.
-    /// The encoder implements encoding using <https://docs.rs/postgres-types/0.2.3/postgres_types/trait.ToSql.html>
     pub fn binary_format(&self, data_type: &DataType) -> to_binary::Result<Bytes> {
+        use self::to_binary::ToBinary;
         self.to_binary_with_type(data_type).transpose().unwrap()
     }
 
@@ -1019,7 +967,7 @@ impl ScalarRefImpl<'_> {
 }
 
 impl ScalarImpl {
-    /// Serialize the scalar.
+    /// Serialize the scalar into the `memcomparable` format.
     pub fn serialize(
         &self,
         ser: &mut memcomparable::Serializer<impl BufMut>,
@@ -1027,7 +975,7 @@ impl ScalarImpl {
         self.as_scalar_ref_impl().serialize(ser)
     }
 
-    /// Deserialize the scalar.
+    /// Deserialize the scalar from the `memcomparable` format.
     pub fn deserialize(
         ty: &DataType,
         de: &mut memcomparable::Deserializer<impl Buf>,
@@ -1061,7 +1009,7 @@ impl ScalarImpl {
             Ty::Timestamptz => Self::Timestamptz(Timestamptz::deserialize(de)?),
             Ty::Date => Self::Date({
                 let days = i32::deserialize(de)?;
-                Date::with_days(days)
+                Date::with_days_since_ce(days)
                     .map_err(|e| memcomparable::Error::Message(e.to_report_string()))?
             }),
             Ty::Jsonb => Self::Jsonb(JsonbVal::memcmp_deserialize(de)?),
diff --git a/src/common/src/types/scalar_impl.rs b/src/common/src/types/scalar_impl.rs
index d222473fa6bbb..43742f74c7b51 100644
--- a/src/common/src/types/scalar_impl.rs
+++ b/src/common/src/types/scalar_impl.rs
@@ -27,7 +27,7 @@ pub trait ScalarPartialOrd: Scalar {
 /// Implement `Scalar` and `ScalarRef` for native type.
 /// For `PrimitiveArrayItemType`, clone is trivial, so `T` is both `Scalar` and `ScalarRef`.
 macro_rules! impl_all_native_scalar {
-    ($({ $scalar_type:ty, $variant_name:ident } ),*) => {
+    ($({ $scalar_type:ty, $_variant_name:ident, $_read_fn:ident } ),*) => {
         $(
             impl Scalar for $scalar_type {
                 type ScalarRefType<'a> = Self;
diff --git a/src/common/src/types/serial.rs b/src/common/src/types/serial.rs
index 5f4ba237ee300..c36fb1eb11ff2 100644
--- a/src/common/src/types/serial.rs
+++ b/src/common/src/types/serial.rs
@@ -24,7 +24,7 @@ use crate::util::row_id::RowId;
 
 // Serial is an alias for i64
 #[derive(Debug, Copy, Clone, PartialEq, Eq, Ord, PartialOrd, Default, Hash)]
-pub struct Serial(i64);
+pub struct Serial(pub(crate) i64);
 
 impl From<Serial> for i64 {
     fn from(value: Serial) -> i64 {
@@ -75,17 +75,6 @@ impl crate::types::to_text::ToText for Serial {
     }
 }
 
-impl crate::types::to_binary::ToBinary for Serial {
-    fn to_binary_with_type(
-        &self,
-        _ty: &crate::types::DataType,
-    ) -> super::to_binary::Result<Option<bytes::Bytes>> {
-        let mut output = bytes::BytesMut::new();
-        self.0.to_sql(&Type::ANY, &mut output).unwrap();
-        Ok(Some(output.freeze()))
-    }
-}
-
 impl ToSql for Serial {
     accepts!(INT8);
 
diff --git a/src/common/src/types/timestamptz.rs b/src/common/src/types/timestamptz.rs
index feafee6e212bf..34a6bd2465271 100644
--- a/src/common/src/types/timestamptz.rs
+++ b/src/common/src/types/timestamptz.rs
@@ -13,17 +13,18 @@
 // limitations under the License.
 
 use std::error::Error;
-use std::io::Write;
+use std::io::{Cursor, Write};
 use std::str::FromStr;
 
-use bytes::{Bytes, BytesMut};
+use anyhow::Context;
+use byteorder::{BigEndian, ReadBytesExt};
+use bytes::BytesMut;
 use chrono::{DateTime, Datelike, TimeZone, Utc};
 use chrono_tz::Tz;
 use postgres_types::{accepts, to_sql_checked, FromSql, IsNull, ToSql, Type};
 use risingwave_common_estimate_size::ZeroHeapSize;
 use serde::{Deserialize, Serialize};
 
-use super::to_binary::ToBinary;
 use super::to_text::ToText;
 use super::DataType;
 use crate::array::ArrayResult;
@@ -65,18 +66,6 @@ impl<'a> FromSql<'a> for Timestamptz {
     }
 }
 
-impl ToBinary for Timestamptz {
-    fn to_binary_with_type(&self, _ty: &DataType) -> super::to_binary::Result<Option<Bytes>> {
-        let instant = self.to_datetime_utc();
-        let mut out = BytesMut::new();
-        // postgres_types::Type::ANY is only used as a placeholder.
-        instant
-            .to_sql(&postgres_types::Type::ANY, &mut out)
-            .unwrap();
-        Ok(Some(out.freeze()))
-    }
-}
-
 impl ToText for Timestamptz {
     fn write<W: std::fmt::Write>(&self, f: &mut W) -> std::fmt::Result {
         // Just a meaningful representation as placeholder. The real implementation depends
@@ -147,8 +136,11 @@ impl Timestamptz {
             .map_err(|_| format!("'{time_zone}' is not a valid timezone"))
     }
 
-    pub fn from_protobuf(timestamp_micros: i64) -> ArrayResult<Self> {
-        Ok(Self(timestamp_micros))
+    pub fn from_protobuf(cur: &mut Cursor<&[u8]>) -> ArrayResult<Timestamptz> {
+        let micros = cur
+            .read_i64::<BigEndian>()
+            .context("failed to read i64 from Timestamptz buffer")?;
+        Ok(Self(micros))
     }
 
     pub fn to_protobuf(self, output: &mut impl Write) -> ArrayResult<usize> {
diff --git a/src/common/src/types/to_binary.rs b/src/common/src/types/to_binary.rs
index 5ab9fd316dcad..56eea301f3f61 100644
--- a/src/common/src/types/to_binary.rs
+++ b/src/common/src/types/to_binary.rs
@@ -15,7 +15,10 @@
 use bytes::{Bytes, BytesMut};
 use postgres_types::{ToSql, Type};
 
-use super::{DataType, DatumRef, ScalarRefImpl, F32, F64};
+use super::{
+    DataType, Date, Decimal, Interval, ScalarRefImpl, Serial, Time, Timestamp, Timestamptz, F32,
+    F64,
+};
 use crate::error::NotImplemented;
 
 /// Error type for [`ToBinary`] trait.
@@ -30,14 +33,15 @@ pub enum ToBinaryError {
 
 pub type Result<T> = std::result::Result<T, ToBinaryError>;
 
-// Used to convert ScalarRef to text format
+/// Converts `ScalarRef` to pgwire "BINARY" format.
+///
+/// [`postgres_types::ToSql`] has similar functionality, and most of our types implement
+/// that trait and forward `ToBinary` to it directly.
 pub trait ToBinary {
     fn to_binary_with_type(&self, ty: &DataType) -> Result<Option<Bytes>>;
 }
-
-// implement use to_sql
 macro_rules! implement_using_to_sql {
-    ($({ $scalar_type:ty, $data_type:ident, $accessor:expr } ),*) => {
+    ($({ $scalar_type:ty, $data_type:ident, $accessor:expr } ),* $(,)?) => {
         $(
             impl ToBinary for $scalar_type {
                 fn to_binary_with_type(&self, ty: &DataType) -> Result<Option<Bytes>> {
@@ -64,7 +68,14 @@ implement_using_to_sql! {
     { F32, Float32, |x: &F32| x.0 },
     { F64, Float64, |x: &F64| x.0 },
     { bool, Boolean, |x| x },
-    { &[u8], Bytea, |x| x }
+    { &[u8], Bytea, |x| x },
+    { Time, Time, |x: &Time| x.0 },
+    { Date, Date, |x: &Date| x.0 },
+    { Timestamp, Timestamp, |x: &Timestamp| x.0 },
+    { Decimal, Decimal, |x| x },
+    { Interval, Interval, |x| x },
+    { Serial, Serial, |x: &Serial| x.0 },
+    { Timestamptz, Timestamptz, |x: &Timestamptz| x.to_datetime_utc() }
 }
 
 impl ToBinary for ScalarRefImpl<'_> {
@@ -94,12 +105,3 @@ impl ToBinary for ScalarRefImpl<'_> {
         }
     }
 }
-
-impl ToBinary for DatumRef<'_> {
-    fn to_binary_with_type(&self, ty: &DataType) -> Result<Option<Bytes>> {
-        match self {
-            Some(scalar) => scalar.to_binary_with_type(ty),
-            None => Ok(None),
-        }
-    }
-}
diff --git a/src/common/src/types/to_text.rs b/src/common/src/types/to_text.rs
index ca140b93c37b7..166356f1977e8 100644
--- a/src/common/src/types/to_text.rs
+++ b/src/common/src/types/to_text.rs
@@ -18,7 +18,24 @@ use std::num::FpCategory;
 use super::{DataType, DatumRef, ScalarRefImpl};
 use crate::dispatch_scalar_ref_variants;
 
-// Used to convert ScalarRef to text format
+/// Converts `ScalarRef` to pgwire "TEXT" format.
+///
+/// ## Relationship with casting to varchar
+///
+/// For most types, this is also the implementation for casting to varchar, but there are exceptions.
+/// e.g., The TEXT format for boolean is `t` / `f` while they cast to varchar `true` / `false`.
+/// - <https://github.com/postgres/postgres/blob/REL_16_3/src/include/catalog/pg_cast.dat#L438-L439>
+/// - <https://www.postgresql.org/docs/16/sql-createcast.html#:~:text=A%20small%20number%20of%20the%20built%2Din%20types%20do%20indeed%20have%20different%20behaviors%20for%20conversions%2C%20mostly%20because%20of%20requirements%20of%20the%20SQL%20standard>
+///
+/// ## Relationship with `ToString`/`Display`
+///
+/// For some types, the implementation diverge from Rust's standard `ToString`/`Display`,
+/// to match PostgreSQL's representation.
+///
+/// ---
+///
+/// FIXME: `ToText` should depend on a lot of other stuff
+/// but we have not implemented them yet: timezone, date style, interval style, bytea output, etc
 pub trait ToText {
     /// Write the text to the writer *regardless* of its data type
     ///
@@ -39,26 +56,10 @@ pub trait ToText {
     /// text. E.g. for Int64, it will convert to text as a Int64 type.
     /// We should prefer to use `to_text_with_type` because it's more clear and readable.
     ///
-    /// Following is the relationship between scalar and default type:
-    /// - `ScalarRefImpl::Int16` -> `DataType::Int16`
-    /// - `ScalarRefImpl::Int32` -> `DataType::Int32`
-    /// - `ScalarRefImpl::Int64` -> `DataType::Int64`
-    /// - `ScalarRefImpl::Int256` -> `DataType::Int256`
-    /// - `ScalarRefImpl::Float32` -> `DataType::Float32`
-    /// - `ScalarRefImpl::Float64` -> `DataType::Float64`
-    /// - `ScalarRefImpl::Decimal` -> `DataType::Decimal`
-    /// - `ScalarRefImpl::Bool` -> `DataType::Boolean`
-    /// - `ScalarRefImpl::Utf8` -> `DataType::Varchar`
-    /// - `ScalarRefImpl::Bytea` -> `DataType::Bytea`
-    /// - `ScalarRefImpl::Date` -> `DataType::Date`
-    /// - `ScalarRefImpl::Time` -> `DataType::Time`
-    /// - `ScalarRefImpl::Timestamp` -> `DataType::Timestamp`
-    /// - `ScalarRefImpl::Timestamptz` -> `DataType::Timestamptz`
-    /// - `ScalarRefImpl::Interval` -> `DataType::Interval`
-    /// - `ScalarRefImpl::Jsonb` -> `DataType::Jsonb`
-    /// - `ScalarRefImpl::List` -> `DataType::List`
-    /// - `ScalarRefImpl::Struct` -> `DataType::Struct`
-    /// - `ScalarRefImpl::Serial` -> `DataType::Serial`
+    /// Note: currently the `DataType` param is actually unnecessary.
+    /// Previously, Timestamptz is also represented as int64, and we need the data type to distinguish them.
+    /// Now we have 1-1 mapping, and it happens to be the case that PostgreSQL default `ToText` format does
+    /// not need additional metadata like field names contained in `DataType`.
     fn to_text(&self) -> String {
         let mut s = String::new();
         self.write(&mut s).unwrap();
diff --git a/src/common/src/util/epoch.rs b/src/common/src/util/epoch.rs
index 56dbdf6c54da9..cba00f4c5ddbd 100644
--- a/src/common/src/util/epoch.rs
+++ b/src/common/src/util/epoch.rs
@@ -77,6 +77,10 @@ impl Epoch {
         Epoch((mi - UNIX_RISINGWAVE_DATE_SEC * 1000) << EPOCH_PHYSICAL_SHIFT_BITS)
     }
 
+    pub fn from_unix_millis_or_earliest(mi: u64) -> Self {
+        Epoch((mi.saturating_sub(UNIX_RISINGWAVE_DATE_SEC * 1000)) << EPOCH_PHYSICAL_SHIFT_BITS)
+    }
+
     pub fn physical_now() -> u64 {
         UNIX_RISINGWAVE_DATE_EPOCH
             .elapsed()
diff --git a/src/common/src/util/stream_graph_visitor.rs b/src/common/src/util/stream_graph_visitor.rs
index 553ede90bcb77..5b990c0186401 100644
--- a/src/common/src/util/stream_graph_visitor.rs
+++ b/src/common/src/util/stream_graph_visitor.rs
@@ -133,12 +133,7 @@ pub fn visit_stream_node_tables_inner<F>(
                 optional!(node.memo_table, "TemporalJoinMemo");
             }
             NodeBody::DynamicFilter(node) => {
-                if node.condition_always_relax {
-                    always!(node.left_table, "DynamicFilterLeftNotSatisfy");
-                } else {
-                    always!(node.left_table, "DynamicFilterLeft");
-                }
-
+                always!(node.left_table, "DynamicFilterLeft");
                 always!(node.right_table, "DynamicFilterRight");
             }
 
@@ -269,6 +264,11 @@ pub fn visit_stream_node_tables_inner<F>(
             NodeBody::Materialize(node) if !internal_tables_only => {
                 always!(node.table, "Materialize")
             }
+
+            NodeBody::GlobalApproxPercentile(node) => {
+                always!(node.bucket_state_table, "GlobalApproxPercentileBucketState");
+                always!(node.count_state_table, "GlobalApproxPercentileCountState");
+            }
             _ => {}
         }
     };
diff --git a/src/common/src/util/value_encoding/column_aware_row_encoding.rs b/src/common/src/util/value_encoding/column_aware_row_encoding.rs
index 1bfbc0c926410..aea36d4b64dac 100644
--- a/src/common/src/util/value_encoding/column_aware_row_encoding.rs
+++ b/src/common/src/util/value_encoding/column_aware_row_encoding.rs
@@ -19,23 +19,15 @@
 //! We have a `Serializer` and a `Deserializer` for each schema of `Row`, which can be reused
 //! until schema changes
 
-use std::collections::{BTreeMap, BTreeSet, HashSet};
+use std::collections::HashSet;
 use std::sync::Arc;
 
+use ahash::HashMap;
 use bitflags::bitflags;
 
 use super::*;
 use crate::catalog::ColumnId;
 
-// deprecated design of have a Width to represent number of datum
-// may be considered should `ColumnId` representation be optimized
-// #[derive(Clone, Copy)]
-// enum Width {
-//     Mid(u8),
-//     Large(u16),
-//     Extra(u32),
-// }
-
 bitflags! {
     #[derive(Debug, Clone, Copy, PartialEq, Eq)]
     struct Flag: u8 {
@@ -176,9 +168,11 @@ impl ValueRowSerializer for Serializer {
 /// Should non-null default values be specified, a new field could be added to Deserializer
 #[derive(Clone)]
 pub struct Deserializer {
-    needed_column_ids: BTreeMap<i32, usize>,
+    required_column_ids: HashMap<i32, usize>,
     schema: Arc<[DataType]>,
-    default_column_values: Vec<(usize, Datum)>,
+
+    /// A row with default values for each column or `None` if no default value is specified
+    default_row: Vec<Datum>,
 }
 
 impl Deserializer {
@@ -188,14 +182,18 @@ impl Deserializer {
         column_with_default: impl Iterator<Item = (usize, Datum)>,
     ) -> Self {
         assert_eq!(column_ids.len(), schema.len());
+        let mut default_row: Vec<Datum> = vec![None; schema.len()];
+        for (i, datum) in column_with_default {
+            default_row[i] = datum;
+        }
         Self {
-            needed_column_ids: column_ids
+            required_column_ids: column_ids
                 .iter()
                 .enumerate()
                 .map(|(i, c)| (c.get_id(), i))
-                .collect::<BTreeMap<_, _>>(),
+                .collect::<HashMap<_, _>>(),
             schema,
-            default_column_values: column_with_default.collect(),
+            default_row,
         }
     }
 }
@@ -214,12 +212,11 @@ impl ValueRowDeserializer for Deserializer {
         let data_start_idx = offsets_start_idx + datum_num * offset_bytes;
         let offsets = &encoded_bytes[offsets_start_idx..data_start_idx];
         let data = &encoded_bytes[data_start_idx..];
-        let mut datums: Vec<Option<Datum>> = vec![None; self.schema.len()];
-        let mut contained_indices = BTreeSet::new();
+
+        let mut row = self.default_row.clone();
         for i in 0..datum_num {
             let this_id = encoded_bytes.get_i32_le();
-            if let Some(&decoded_idx) = self.needed_column_ids.get(&this_id) {
-                contained_indices.insert(decoded_idx);
+            if let Some(&decoded_idx) = self.required_column_ids.get(&this_id) {
                 let this_offset_start_idx = i * offset_bytes;
                 let mut this_offset_slice =
                     &offsets[this_offset_start_idx..(this_offset_start_idx + offset_bytes)];
@@ -246,15 +243,10 @@ impl ValueRowDeserializer for Deserializer {
                         &mut data_slice,
                     )?)
                 };
-                datums[decoded_idx] = Some(data);
-            }
-        }
-        for (id, datum) in &self.default_column_values {
-            if !contained_indices.contains(id) {
-                datums[*id].get_or_insert(datum.clone());
+                row[decoded_idx] = data;
             }
         }
-        Ok(datums.into_iter().map(|d| d.unwrap_or(None)).collect())
+        Ok(row)
     }
 }
 
diff --git a/src/common/src/util/value_encoding/mod.rs b/src/common/src/util/value_encoding/mod.rs
index a3da88911ad9a..322c542557ed7 100644
--- a/src/common/src/util/value_encoding/mod.rs
+++ b/src/common/src/util/value_encoding/mod.rs
@@ -428,7 +428,7 @@ fn deserialize_timestamp(data: &mut impl Buf) -> Result<Timestamp> {
 
 fn deserialize_date(data: &mut impl Buf) -> Result<Date> {
     let days = data.get_i32_le();
-    Date::with_days(days).map_err(|_e| ValueEncodingError::InvalidDateEncoding(days))
+    Date::with_days_since_ce(days).map_err(|_e| ValueEncodingError::InvalidDateEncoding(days))
 }
 
 fn deserialize_decimal(data: &mut impl Buf) -> Result<Decimal> {
diff --git a/src/common/src/vnode_mapping/vnode_placement.rs b/src/common/src/vnode_mapping/vnode_placement.rs
index 036cfebe792bb..aea08a1b74352 100644
--- a/src/common/src/vnode_mapping/vnode_placement.rs
+++ b/src/common/src/vnode_mapping/vnode_placement.rs
@@ -20,7 +20,7 @@ use num_integer::Integer;
 use risingwave_common::hash::WorkerSlotId;
 use risingwave_pb::common::WorkerNode;
 
-use crate::buffer::{Bitmap, BitmapBuilder};
+use crate::bitmap::{Bitmap, BitmapBuilder};
 use crate::hash::{VirtualNode, WorkerSlotMapping};
 
 /// Calculate a new vnode mapping, keeping locality and balance on a best effort basis.
@@ -37,7 +37,7 @@ pub fn place_vnode(
         .iter()
         .filter(|w| w.property.as_ref().map_or(false, |p| p.is_serving))
         .sorted_by_key(|w| w.id)
-        .map(|w| (0..w.parallel_units.len()).map(|idx| WorkerSlotId::new(w.id, idx)))
+        .map(|w| (0..w.parallelism()).map(|idx| WorkerSlotId::new(w.id, idx)))
         .collect();
 
     // Set serving parallelism to the minimum of total number of worker slots, specified
@@ -198,42 +198,23 @@ pub fn place_vnode(
 
 #[cfg(test)]
 mod tests {
-    use std::collections::HashMap;
 
     use risingwave_common::hash::WorkerSlotMapping;
     use risingwave_pb::common::worker_node::Property;
-    use risingwave_pb::common::{ParallelUnit, WorkerNode};
+    use risingwave_pb::common::WorkerNode;
 
-    use crate::hash::{ParallelUnitId, VirtualNode};
+    use crate::hash::VirtualNode;
     use crate::vnode_mapping::vnode_placement::place_vnode;
     #[test]
     fn test_place_vnode() {
         assert_eq!(VirtualNode::COUNT, 256);
 
-        let mut pu_id_counter: ParallelUnitId = 0;
-        let mut pu_to_worker: HashMap<ParallelUnitId, u32> = Default::default();
         let serving_property = Property {
             is_unschedulable: false,
             is_serving: true,
             is_streaming: false,
         };
 
-        let mut gen_pus_for_worker =
-            |worker_node_id: u32, number: u32, pu_to_worker: &mut HashMap<ParallelUnitId, u32>| {
-                let mut results = vec![];
-                for i in 0..number {
-                    results.push(ParallelUnit {
-                        id: pu_id_counter + i,
-                        worker_node_id,
-                    })
-                }
-                pu_id_counter += number;
-                for pu in &results {
-                    pu_to_worker.insert(pu.id, pu.worker_node_id);
-                }
-                results
-            };
-
         let count_same_vnode_mapping = |wm1: &WorkerSlotMapping, wm2: &WorkerSlotMapping| {
             assert_eq!(wm1.len(), 256);
             assert_eq!(wm2.len(), 256);
@@ -249,7 +230,7 @@ mod tests {
 
         let worker_1 = WorkerNode {
             id: 1,
-            parallel_units: gen_pus_for_worker(1, 1, &mut pu_to_worker),
+            parallelism: 1,
             property: Some(serving_property.clone()),
             ..Default::default()
         };
@@ -264,7 +245,7 @@ mod tests {
 
         let worker_2 = WorkerNode {
             id: 2,
-            parallel_units: gen_pus_for_worker(2, 50, &mut pu_to_worker),
+            parallelism: 50,
             property: Some(serving_property.clone()),
             ..Default::default()
         };
@@ -283,7 +264,7 @@ mod tests {
 
         let worker_3 = WorkerNode {
             id: 3,
-            parallel_units: gen_pus_for_worker(3, 60, &mut pu_to_worker),
+            parallelism: 60,
             property: Some(serving_property),
             ..Default::default()
         };
diff --git a/src/compute/Cargo.toml b/src/compute/Cargo.toml
index 817578db32631..a3f74792982f2 100644
--- a/src/compute/Cargo.toml
+++ b/src/compute/Cargo.toml
@@ -58,6 +58,7 @@ tokio-stream = "0.1"
 tonic = { workspace = true }
 tower = { version = "0.4", features = ["util", "load-shed"] }
 tracing = "0.1"
+uuid = { version = "1.8.0", features = ["v4"] }
 
 [target.'cfg(not(madsim))'.dependencies]
 workspace-hack = { path = "../workspace-hack" }
diff --git a/src/compute/src/lib.rs b/src/compute/src/lib.rs
index 7a3d2be65d1df..d91fb56b1cb88 100644
--- a/src/compute/src/lib.rs
+++ b/src/compute/src/lib.rs
@@ -18,7 +18,6 @@
 #![feature(let_chains)]
 #![feature(lint_reasons)]
 #![feature(impl_trait_in_assoc_type)]
-#![feature(lazy_cell)]
 #![cfg_attr(coverage, feature(coverage_attribute))]
 
 #[macro_use]
@@ -136,6 +135,15 @@ pub struct ComputeNodeOpts {
     #[deprecated = "connector node has been deprecated."]
     #[clap(long, hide = true, env = "RW_CONNECTOR_RPC_ENDPOINT")]
     pub connector_rpc_endpoint: Option<String>,
+
+    /// The path of the temp secret file directory.
+    #[clap(
+        long,
+        hide = true,
+        env = "RW_TEMP_SECRET_FILE_DIR",
+        default_value = "./secrets"
+    )]
+    pub temp_secret_file_dir: String,
 }
 
 impl risingwave_common::opts::Opts for ComputeNodeOpts {
diff --git a/src/compute/src/memory/config.rs b/src/compute/src/memory/config.rs
index 2a6607c268a67..234490773f40f 100644
--- a/src/compute/src/memory/config.rs
+++ b/src/compute/src/memory/config.rs
@@ -168,6 +168,24 @@ pub fn storage_memory_config(
         ((non_reserved_memory_bytes as f64 * compactor_memory_proportion).ceil() as usize) >> 20,
     );
 
+    // The file cache flush buffer threshold is used as a emergency limitation.
+    // On most cases the flush buffer is not supposed to be as large as the threshold.
+    // So, the file cache flush buffer threshold size is not calculated in the memory usage.
+    let block_file_cache_flush_buffer_threshold_mb = storage_config
+        .data_file_cache
+        .flush_buffer_threshold_mb
+        .unwrap_or(
+            risingwave_common::config::default::storage::block_file_cache_flush_buffer_threshold_mb(
+            ),
+        );
+    let meta_file_cache_flush_buffer_threshold_mb = storage_config
+        .meta_file_cache
+        .flush_buffer_threshold_mb
+        .unwrap_or(
+            risingwave_common::config::default::storage::meta_file_cache_flush_buffer_threshold_mb(
+            ),
+        );
+
     let total_calculated_mb = block_cache_capacity_mb
         + meta_cache_capacity_mb
         + shared_buffer_capacity_mb
@@ -276,6 +294,8 @@ pub fn storage_memory_config(
         prefetch_buffer_capacity_mb,
         block_cache_eviction_config,
         meta_cache_eviction_config,
+        block_file_cache_flush_buffer_threshold_mb,
+        meta_file_cache_flush_buffer_threshold_mb,
     }
 }
 
diff --git a/src/compute/src/observer/observer_manager.rs b/src/compute/src/observer/observer_manager.rs
index e4192fb0b6f9c..c028c1e851613 100644
--- a/src/compute/src/observer/observer_manager.rs
+++ b/src/compute/src/observer/observer_manager.rs
@@ -12,9 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use risingwave_common::secret::LocalSecretManager;
 use risingwave_common::system_param::local_manager::LocalSystemParamsManagerRef;
-use risingwave_common_service::observer_manager::{ObserverState, SubscribeCompute};
-use risingwave_pb::meta::subscribe_response::Info;
+use risingwave_common_service::ObserverState;
+use risingwave_pb::meta::subscribe_response::{Info, Operation};
 use risingwave_pb::meta::SubscribeResponse;
 
 pub struct ComputeObserverNode {
@@ -22,22 +23,38 @@ pub struct ComputeObserverNode {
 }
 
 impl ObserverState for ComputeObserverNode {
-    type SubscribeType = SubscribeCompute;
+    fn subscribe_type() -> risingwave_pb::meta::SubscribeType {
+        risingwave_pb::meta::SubscribeType::Compute
+    }
 
     fn handle_notification(&mut self, resp: SubscribeResponse) {
-        let Some(info) = resp.info.as_ref() else {
-            return;
-        };
-
-        match info.to_owned() {
-            Info::SystemParams(p) => self.system_params_manager.try_set_params(p),
-            _ => {
-                panic!("error type notification");
+        if let Some(info) = resp.info.as_ref() {
+            match info.to_owned() {
+                Info::SystemParams(p) => self.system_params_manager.try_set_params(p),
+                Info::Secret(s) => match resp.operation() {
+                    Operation::Add => {
+                        LocalSecretManager::global().add_secret(s.id, s.value);
+                    }
+                    Operation::Delete => {
+                        LocalSecretManager::global().remove_secret(s.id);
+                    }
+                    _ => {
+                        panic!("error type notification");
+                    }
+                },
+                _ => {
+                    panic!("error type notification");
+                }
             }
-        }
+        };
     }
 
-    fn handle_initialization_notification(&mut self, _resp: SubscribeResponse) {}
+    fn handle_initialization_notification(&mut self, resp: SubscribeResponse) {
+        let Some(Info::Snapshot(snapshot)) = resp.info else {
+            unreachable!();
+        };
+        LocalSecretManager::global().init_secrets(snapshot.secrets);
+    }
 }
 
 impl ComputeObserverNode {
diff --git a/src/compute/src/rpc/service/exchange_service.rs b/src/compute/src/rpc/service/exchange_service.rs
index f44f6f7552bb3..e4082a88ea9e6 100644
--- a/src/compute/src/rpc/service/exchange_service.rs
+++ b/src/compute/src/rpc/service/exchange_service.rs
@@ -24,7 +24,7 @@ use risingwave_pb::task_service::{
     permits, GetDataRequest, GetDataResponse, GetStreamRequest, GetStreamResponse, PbPermits,
 };
 use risingwave_stream::executor::exchange::permit::{MessageWithPermits, Receiver};
-use risingwave_stream::executor::Message;
+use risingwave_stream::executor::DispatcherMessage;
 use risingwave_stream::task::LocalStreamManager;
 use thiserror_ext::AsReport;
 use tokio_stream::wrappers::ReceiverStream;
@@ -169,21 +169,14 @@ impl ExchangeServiceImpl {
                 Either::Left(permits_to_add) => {
                     permits.add_permits(permits_to_add);
                 }
-                Either::Right(MessageWithPermits {
-                    mut message,
-                    permits,
-                }) => {
-                    // Erase the mutation of the barrier to avoid decoding in remote side.
-                    if let Message::Barrier(barrier) = &mut message {
-                        barrier.mutation = None;
-                    }
+                Either::Right(MessageWithPermits { message, permits }) => {
                     let proto = message.to_protobuf();
                     // forward the acquired permit to the downstream
                     let response = GetStreamResponse {
                         message: Some(proto),
                         permits: Some(PbPermits { value: permits }),
                     };
-                    let bytes = Message::get_encoded_len(&response);
+                    let bytes = DispatcherMessage::get_encoded_len(&response);
 
                     yield response;
 
diff --git a/src/compute/src/server.rs b/src/compute/src/server.rs
index 3b0466b6cc0d1..3270897c29d31 100644
--- a/src/compute/src/server.rs
+++ b/src/compute/src/server.rs
@@ -28,7 +28,8 @@ use risingwave_common::config::{
     MAX_CONNECTION_WINDOW_SIZE, STREAM_WINDOW_SIZE,
 };
 use risingwave_common::lru::init_global_sequencer_args;
-use risingwave_common::monitor::connection::{RouterExt, TcpConfig};
+use risingwave_common::monitor::{RouterExt, TcpConfig};
+use risingwave_common::secret::LocalSecretManager;
 use risingwave_common::system_param::local_manager::LocalSystemParamsManager;
 use risingwave_common::system_param::reader::SystemParamsRead;
 use risingwave_common::telemetry::manager::TelemetryManager;
@@ -38,9 +39,7 @@ use risingwave_common::util::pretty_bytes::convert;
 use risingwave_common::util::tokio_util::sync::CancellationToken;
 use risingwave_common::{GIT_SHA, RW_VERSION};
 use risingwave_common_heap_profiling::HeapProfiler;
-use risingwave_common_service::metrics_manager::MetricsManager;
-use risingwave_common_service::observer_manager::ObserverManager;
-use risingwave_common_service::tracing::TracingExtractLayer;
+use risingwave_common_service::{MetricsManager, ObserverManager, TracingExtractLayer};
 use risingwave_connector::source::monitor::GLOBAL_SOURCE_METRICS;
 use risingwave_dml::dml_manager::DmlManager;
 use risingwave_pb::common::WorkerType;
@@ -217,6 +216,12 @@ pub async fn compute_node_serve(
     .await
     .unwrap();
 
+    LocalSecretManager::init(
+        opts.temp_secret_file_dir,
+        meta_client.cluster_id().to_string(),
+        worker_id,
+    );
+
     // Initialize observer manager.
     let system_params_manager = Arc::new(LocalSystemParamsManager::new(system_params.clone()));
     let compute_observer_node = ComputeObserverNode::new(system_params_manager.clone());
@@ -340,7 +345,9 @@ pub async fn compute_node_serve(
     ));
 
     // Initialize batch environment.
-    let client_pool = Arc::new(ComputeClientPool::new(config.server.connection_pool_size));
+    let batch_client_pool = Arc::new(ComputeClientPool::new(
+        config.batch_exchange_connection_pool_size(),
+    ));
     let batch_env = BatchEnvironment::new(
         batch_mgr.clone(),
         advertise_addr.clone(),
@@ -349,7 +356,7 @@ pub async fn compute_node_serve(
         state_store.clone(),
         batch_task_metrics.clone(),
         batch_executor_metrics.clone(),
-        client_pool,
+        batch_client_pool,
         dml_mgr.clone(),
         source_metrics.clone(),
         batch_spill_metrics.clone(),
@@ -357,6 +364,9 @@ pub async fn compute_node_serve(
     );
 
     // Initialize the streaming environment.
+    let stream_client_pool = Arc::new(ComputeClientPool::new(
+        config.streaming_exchange_connection_pool_size(),
+    ));
     let stream_env = StreamEnvironment::new(
         advertise_addr.clone(),
         stream_config,
@@ -366,6 +376,7 @@ pub async fn compute_node_serve(
         system_params_manager.clone(),
         source_metrics,
         meta_client.clone(),
+        stream_client_pool,
     );
 
     let stream_mgr = LocalStreamManager::new(
@@ -394,7 +405,7 @@ pub async fn compute_node_serve(
         meta_cache,
         block_cache,
     );
-    let config_srv = ConfigServiceImpl::new(batch_mgr, stream_mgr);
+    let config_srv = ConfigServiceImpl::new(batch_mgr, stream_mgr.clone());
     let health_srv = HealthServiceImpl::new();
 
     let telemetry_manager = TelemetryManager::new(
@@ -412,7 +423,9 @@ pub async fn compute_node_serve(
 
     // Clean up the spill directory.
     #[cfg(not(madsim))]
-    SpillOp::clean_spill_directory().await.unwrap();
+    if config.batch.enable_spill {
+        SpillOp::clean_spill_directory().await.unwrap();
+    }
 
     let server = tonic::transport::Server::builder()
         .initial_connection_window_size(MAX_CONNECTION_WINDOW_SIZE)
@@ -458,8 +471,12 @@ pub async fn compute_node_serve(
     // Wait for the shutdown signal.
     shutdown.cancelled().await;
 
-    // TODO(shutdown): gracefully unregister from the meta service (need to cautious since it may
-    // trigger auto-scaling)
+    // Unregister from the meta service, then...
+    // - batch queries will not be scheduled to this compute node,
+    // - streaming actors will not be scheduled to this compute node after next recovery.
+    meta_client.try_unregister().await;
+    // Shutdown the streaming manager.
+    let _ = stream_mgr.shutdown().await;
 
     // NOTE(shutdown): We can't simply join the tonic server here because it only returns when all
     // existing connections are closed, while we have long-running streaming calls that never
diff --git a/src/compute/tests/cdc_tests.rs b/src/compute/tests/cdc_tests.rs
index 26f822bae558f..974cb15259e2a 100644
--- a/src/compute/tests/cdc_tests.rs
+++ b/src/compute/tests/cdc_tests.rs
@@ -79,6 +79,8 @@ impl MockOffsetGenExecutor {
                 lsn: None,
                 txid: None,
                 tx_usec: None,
+                change_lsn: None,
+                commit_lsn: None,
             },
             is_heartbeat: false,
         };
@@ -311,6 +313,7 @@ async fn test_cdc_backfill() -> StreamResult<()> {
             added_actors: HashSet::new(),
             splits,
             pause: false,
+            subscriptions_to_add: vec![],
         }));
 
     tx.send_barrier(init_barrier);
@@ -381,6 +384,7 @@ async fn test_cdc_backfill() -> StreamResult<()> {
         "RowSeqExecutor2".to_string(),
         None,
         None,
+        None,
     ));
 
     // check result
diff --git a/src/compute/tests/integration_tests.rs b/src/compute/tests/integration_tests.rs
index 14dbfed466158..11cbe06386a29 100644
--- a/src/compute/tests/integration_tests.rs
+++ b/src/compute/tests/integration_tests.rs
@@ -28,7 +28,7 @@ use risingwave_batch::executor::{
     RowSeqScanExecutor, ScanRange,
 };
 use risingwave_common::array::{Array, DataChunk, F64Array, SerialArray};
-use risingwave_common::buffer::Bitmap;
+use risingwave_common::bitmap::Bitmap;
 use risingwave_common::catalog::{
     ColumnDesc, ColumnId, ConflictBehavior, Field, Schema, TableId, INITIAL_TABLE_VERSION_ID,
 };
@@ -267,6 +267,7 @@ async fn test_table_materialize() -> StreamResult<()> {
         "RowSeqExecutor2".to_string(),
         None,
         None,
+        None,
     ));
     let mut stream = scan.execute();
     let result = stream.next().await;
@@ -337,6 +338,7 @@ async fn test_table_materialize() -> StreamResult<()> {
         "RowSeqScanExecutor2".to_string(),
         None,
         None,
+        None,
     ));
 
     let mut stream = scan.execute();
@@ -416,6 +418,7 @@ async fn test_table_materialize() -> StreamResult<()> {
         "RowSeqScanExecutor2".to_string(),
         None,
         None,
+        None,
     ));
 
     let mut stream = scan.execute();
@@ -487,6 +490,7 @@ async fn test_row_seq_scan() -> StreamResult<()> {
         "RowSeqScanExecutor2".to_string(),
         None,
         None,
+        None,
     ));
 
     assert_eq!(executor.schema().fields().len(), 3);
diff --git a/src/config/ci-time-travel.toml b/src/config/ci-time-travel.toml
new file mode 100644
index 0000000000000..a5aadc196f9b5
--- /dev/null
+++ b/src/config/ci-time-travel.toml
@@ -0,0 +1,6 @@
+[meta]
+hummock_time_travel_snapshot_interval = 30
+min_sst_retention_time_sec = 1
+
+[system]
+time_travel_retention_ms = 300000
diff --git a/src/config/docs.md b/src/config/docs.md
index f93700f5c5e65..47905d71e5e0c 100644
--- a/src/config/docs.md
+++ b/src/config/docs.md
@@ -39,6 +39,7 @@ This page is automatically generated by `./risedev generate-example-config`
 | event_log_channel_max_size | Keeps the latest N events per channel. | 10 |
 | event_log_enabled |  | true |
 | full_gc_interval_sec | Interval of automatic hummock full GC. | 86400 |
+| hummock_time_travel_snapshot_interval | The interval at which a Hummock version snapshot is taken for time travel. Larger value indicates less storage overhead but worse query performance. | 100 |
 | hummock_version_checkpoint_interval_sec | Interval of hummock version checkpoint. | 30 |
 | hybrid_partition_vnode_count | Count of partitions of tables in default group and materialized view group. The meta node will decide according to some strategy whether to cut the boundaries of the file according to the vnode alignment. Each partition contains aligned data of `VirtualNode::COUNT / hybrid_partition_vnode_count` consecutive virtual-nodes of one state table. Set it zero to disable this feature. | 4 |
 | max_heartbeat_interval_secs | Maximum allowed heartbeat interval in seconds. | 60 |
@@ -57,12 +58,11 @@ This page is automatically generated by `./risedev generate-example-config`
 | periodic_split_compact_group_interval_sec |  | 10 |
 | periodic_tombstone_reclaim_compaction_interval_sec |  | 600 |
 | periodic_ttl_reclaim_compaction_interval_sec | Schedule `ttl_reclaim` compaction for all compaction groups with this interval. | 1800 |
-| secret_store_private_key |  | [100, 101, 109, 111, 45, 115, 101, 99, 114, 101, 116, 45, 112, 114, 105, 118, 97, 116, 101, 45, 107, 101, 121] |
 | split_group_size_limit |  | 68719476736 |
 | table_write_throughput_threshold | The threshold of write throughput to trigger a group split. Increase this configuration value to avoid split too many groups with few data write. | 16777216 |
 | unrecognized |  |  |
 | vacuum_interval_sec | Interval of invoking a vacuum job, to remove stale metadata from meta store and objects from object store. | 30 |
-| vacuum_spin_interval_ms | The spin interval inside a vacuum job. It avoids the vacuum job monopolizing resources of meta node. | 10 |
+| vacuum_spin_interval_ms | The spin interval inside a vacuum job. It avoids the vacuum job monopolizing resources of meta node. | 200 |
 
 ## meta.compaction_config
 
@@ -89,7 +89,7 @@ This page is automatically generated by `./risedev generate-example-config`
 
 | Config | Description | Default |
 |--------|-------------|---------|
-| connection_pool_size |  | 16 |
+| connection_pool_size | The default number of the connections when connecting to a gRPC server. For the connections used in streaming or batch exchange, please refer to the entries in `[stream.developer]` and `[batch.developer]` sections. This value will be used if they are not specified. | 16 |
 | grpc_max_reset_stream |  | 200 |
 | heap_profiling | Enable heap profile dump when memory usage is high. |  |
 | heartbeat_interval_ms | The interval for periodic heartbeat from worker to the meta service. | 1000 |
@@ -169,4 +169,5 @@ This page is automatically generated by `./risedev generate-example-config`
 | pause_on_next_bootstrap | Whether to pause all data sources on next bootstrap. | false |
 | sstable_size_mb | Target size of the Sstable. | 256 |
 | state_store | URL for the state store |  |
+| time_travel_retention_ms | The data retention period for time travel, where 0 indicates that it's disabled. | 0 |
 | use_new_object_prefix_strategy | Whether to split object prefix. |  |
diff --git a/src/config/example.toml b/src/config/example.toml
index 3a394eada8645..866a56017982e 100644
--- a/src/config/example.toml
+++ b/src/config/example.toml
@@ -19,9 +19,10 @@ full_gc_interval_sec = 86400
 collect_gc_watermark_spin_interval_sec = 5
 periodic_compaction_interval_sec = 60
 vacuum_interval_sec = 30
-vacuum_spin_interval_ms = 10
+vacuum_spin_interval_ms = 200
 hummock_version_checkpoint_interval_sec = 30
 enable_hummock_data_archive = false
+hummock_time_travel_snapshot_interval = 100
 min_delta_log_num_for_hummock_version_checkpoint = 10
 max_heartbeat_interval_secs = 60
 disable_recovery = false
@@ -54,7 +55,6 @@ compact_task_table_size_partition_threshold_high = 536870912
 event_log_enabled = true
 event_log_channel_max_size = 10
 enable_dropped_column_reclaim = false
-secret_store_private_key = [100, 101, 109, 111, 45, 115, 101, 99, 114, 101, 116, 45, 112, 114, 105, 118, 97, 116, 101, 45, 107, 101, 121]
 
 [meta.compaction_config]
 max_bytes_for_level_base = 536870912
@@ -123,6 +123,7 @@ stream_memory_controller_sequence_tls_lag = 32
 stream_enable_arrangement_backfill = true
 stream_high_join_amplification_threshold = 2048
 stream_enable_actor_tokio_metrics = false
+stream_exchange_connection_pool_size = 1
 
 [storage]
 share_buffers_sync_parallelism = 1
@@ -195,6 +196,7 @@ recent_filter_rotate_interval_ms = 10000
 set_atomic_write_dir = false
 opendal_upload_concurrency = 8
 opendal_writer_abort_on_err = false
+upload_part_size = 16777216
 
 [storage.object_store.retry]
 req_backoff_interval_ms = 1000
@@ -240,3 +242,4 @@ max_concurrent_creating_streaming_jobs = 1
 pause_on_next_bootstrap = false
 enable_tracing = false
 license_key = ""
+time_travel_retention_ms = 0
diff --git a/src/connector/Cargo.toml b/src/connector/Cargo.toml
index 86491ae464a3f..9647c95d05d95 100644
--- a/src/connector/Cargo.toml
+++ b/src/connector/Cargo.toml
@@ -33,6 +33,7 @@ aws-config = { workspace = true }
 aws-credential-types = { workspace = true }
 aws-msk-iam-sasl-signer = "1.0.0"
 aws-sdk-dynamodb = "1"
+aws-sdk-glue = { workspace = true }
 aws-sdk-kinesis = { workspace = true }
 aws-sdk-s3 = { workspace = true }
 aws-smithy-http = { workspace = true }
@@ -84,7 +85,7 @@ mysql_common = { version = "0.32", default-features = false, features = [
 ] }
 nexmark = { version = "0.2", features = ["serde"] }
 num-bigint = "0.4"
-opendal = { version = "0.47", features = [
+opendal = { workspace = true, features = [
     "executors-tokio",
     "services-fs",
     "services-gcs",
@@ -146,8 +147,19 @@ strum = "0.26"
 strum_macros = "0.26"
 tempfile = "3"
 thiserror = "1"
+
 thiserror-ext = { workspace = true }
-tiberius = { version = "0.12", default-features = false, features = ["chrono", "time", "tds73", "rust_decimal", "bigdecimal", "rustls"] }
+# To easiy get the type_name and impl IntoSql for rust_decimal, we fork the crate.
+# Another reason is that we are planning to refactor their IntoSql trait to allow specifying the type to convert.
+tiberius = { git = "https://github.com/risingwavelabs/tiberius.git", rev = "f834f2deeb9e2fb08afaf73865f330cf31a3876a", default-features = false, features = [
+    "chrono",
+    "sql-browser-tokio",
+    "vendored-openssl",
+    "rust_decimal",
+    "bigdecimal",
+    "tds73",
+    "time",
+] }
 time = "0.3.30"
 tokio = { version = "0.2", package = "madsim-tokio", features = [
     "rt",
diff --git a/src/connector/benches/nexmark_integration.rs b/src/connector/benches/nexmark_integration.rs
index 172931562efef..f59ae08a9ec14 100644
--- a/src/connector/benches/nexmark_integration.rs
+++ b/src/connector/benches/nexmark_integration.rs
@@ -18,8 +18,6 @@
 //! `ByteStreamSourceParserImpl::create` based on the given configuration, rather
 //! than depending on a specific internal implementation.
 
-#![feature(lazy_cell)]
-
 use std::sync::LazyLock;
 
 use criterion::{criterion_group, criterion_main, BatchSize, Criterion};
diff --git a/src/connector/codec/Cargo.toml b/src/connector/codec/Cargo.toml
index d9e6e274f0fef..5086549f4bf4c 100644
--- a/src/connector/codec/Cargo.toml
+++ b/src/connector/codec/Cargo.toml
@@ -35,6 +35,10 @@ thiserror-ext = { workspace = true }
 time = "0.3.30"
 tracing = "0.1"
 
+[dev-dependencies]
+expect-test = "1"
+hex = "0.4"
+
 [target.'cfg(not(madsim))'.dependencies]
 workspace-hack = { path = "../../workspace-hack" }
 
diff --git a/src/connector/codec/src/decoder/avro/mod.rs b/src/connector/codec/src/decoder/avro/mod.rs
index cdd9aea416c8f..93d16a32508db 100644
--- a/src/connector/codec/src/decoder/avro/mod.rs
+++ b/src/connector/codec/src/decoder/avro/mod.rs
@@ -15,7 +15,7 @@
 mod schema;
 use std::sync::LazyLock;
 
-use apache_avro::schema::{DecimalSchema, RecordSchema};
+use apache_avro::schema::{DecimalSchema, RecordSchema, UnionSchema};
 use apache_avro::types::{Value, ValueKind};
 use apache_avro::{Decimal as AvroDecimal, Schema};
 use chrono::Datelike;
@@ -33,6 +33,7 @@ use risingwave_common::util::iter_util::ZipEqFast;
 pub use self::schema::{avro_schema_to_column_descs, MapHandling, ResolvedAvroSchema};
 use super::utils::extract_decimal;
 use super::{bail_uncategorized, uncategorized, Access, AccessError, AccessResult};
+use crate::decoder::avro::schema::avro_schema_to_struct_field_name;
 
 #[derive(Clone)]
 /// Options for parsing an `AvroValue` into Datum, with an optional avro schema.
@@ -107,6 +108,54 @@ impl<'a> AvroParseOptions<'a> {
 
         let v: ScalarImpl = match (type_expected, value) {
             (_, Value::Null) => return Ok(DatumCow::NULL),
+            // ---- Union (with >=2 non null variants), and nullable Union ([null, record]) -----
+            (DataType::Struct(struct_type_info), Value::Union(variant, v)) => {
+                let Some(Schema::Union(u)) = self.schema else {
+                    // XXX: Is this branch actually unreachable? (if self.schema is correctly used)
+                    return Err(create_error());
+                };
+
+                if let Some(inner) = get_nullable_union_inner(u) {
+                    // nullable Union ([null, record])
+                    return Self {
+                        schema: Some(inner),
+                        relax_numeric: self.relax_numeric,
+                    }
+                    .convert_to_datum(v, type_expected);
+                }
+                let variant_schema = &u.variants()[*variant as usize];
+
+                if matches!(variant_schema, &Schema::Null) {
+                    return Ok(DatumCow::NULL);
+                }
+
+                // Here we compare the field name, instead of using the variant idx to find the field idx.
+                // The latter approach might also work, but might be more error-prone.
+                // We will need to get the index of the "null" variant, and then re-map the variant index to the field index.
+                // XXX: probably we can unwrap here (if self.schema is correctly used)
+                let expected_field_name = avro_schema_to_struct_field_name(variant_schema)?;
+
+                let mut fields = Vec::with_capacity(struct_type_info.len());
+                for (field_name, field_type) in struct_type_info
+                    .names()
+                    .zip_eq_fast(struct_type_info.types())
+                {
+                    if field_name == expected_field_name {
+                        let datum = Self {
+                            schema: Some(variant_schema),
+                            relax_numeric: self.relax_numeric,
+                        }
+                        .convert_to_datum(v, field_type)?
+                        .to_owned_datum();
+
+                        fields.push(datum)
+                    } else {
+                        fields.push(None)
+                    }
+                }
+                StructValue::new(fields).into()
+            }
+            // nullable Union ([null, T])
             (_, Value::Union(_, v)) => {
                 let schema = self.extract_inner_schema(None);
                 return Self {
@@ -186,9 +235,11 @@ impl<'a> AvroParseOptions<'a> {
                 .map_err(|_| create_error())?
                 .into(),
             // ---- Date -----
-            (DataType::Date, Value::Date(days)) => Date::with_days(days + unix_epoch_days())
-                .map_err(|_| create_error())?
-                .into(),
+            (DataType::Date, Value::Date(days)) => {
+                Date::with_days_since_ce(days + unix_epoch_days())
+                    .map_err(|_| create_error())?
+                    .into()
+            }
             // ---- Varchar -----
             (DataType::Varchar, Value::Enum(_, symbol)) => borrowed!(symbol.as_str()),
             (DataType::Varchar, Value::String(s)) => borrowed!(s.as_str()),
@@ -290,6 +341,12 @@ impl Access for AvroAccess<'_> {
         let mut value = self.value;
         let mut options: AvroParseOptions<'_> = self.options.clone();
 
+        debug_assert!(
+            path.len() == 1
+                || (path.len() == 2 && matches!(path[0], "before" | "after" | "source")),
+            "unexpected path access: {:?}",
+            path
+        );
         let mut i = 0;
         while i < path.len() {
             let key = path[i];
@@ -299,6 +356,29 @@ impl Access for AvroAccess<'_> {
             };
             match value {
                 Value::Union(_, v) => {
+                    // The debezium "before" field is a nullable union.
+                    // "fields": [
+                    // {
+                    //     "name": "before",
+                    //     "type": [
+                    //         "null",
+                    //         {
+                    //             "type": "record",
+                    //             "name": "Value",
+                    //             "fields": [...],
+                    //         }
+                    //     ],
+                    //     "default": null
+                    // },
+                    // {
+                    //     "name": "after",
+                    //     "type": [
+                    //         "null",
+                    //         "Value"
+                    //     ],
+                    //     "default": null
+                    // },
+                    // ...]
                     value = v;
                     options.schema = options.extract_inner_schema(None);
                     continue;
@@ -338,18 +418,30 @@ pub(crate) fn avro_decimal_to_rust_decimal(
     ))
 }
 
-pub fn avro_schema_skip_union(schema: &Schema) -> anyhow::Result<&Schema> {
+/// If the union schema is `[null, T]` or `[T, null]`, returns `Some(T)`; otherwise returns `None`.
+fn get_nullable_union_inner(union_schema: &UnionSchema) -> Option<&'_ Schema> {
+    let variants = union_schema.variants();
+    // Note: `[null, null] is invalid`, we don't need to worry about that.
+    if variants.len() == 2 && variants.contains(&Schema::Null) {
+        let inner_schema = variants
+            .iter()
+            .find(|s| !matches!(s, &&Schema::Null))
+            .unwrap();
+        Some(inner_schema)
+    } else {
+        None
+    }
+}
+
+pub fn avro_schema_skip_nullable_union(schema: &Schema) -> anyhow::Result<&Schema> {
     match schema {
-        Schema::Union(union_schema) => {
-            let inner_schema = union_schema
-                .variants()
-                .iter()
-                .find(|s| !matches!(s, &&Schema::Null))
-                .ok_or_else(|| {
-                    anyhow::format_err!("illegal avro record schema {:?}", union_schema)
-                })?;
-            Ok(inner_schema)
-        }
+        Schema::Union(union_schema) => match get_nullable_union_inner(union_schema) {
+            Some(s) => Ok(s),
+            None => Err(anyhow::format_err!(
+                "illegal avro union schema, expected [null, T], got {:?}",
+                union_schema
+            )),
+        },
         other => Ok(other),
     }
 }
@@ -372,7 +464,9 @@ pub fn avro_extract_field_schema<'a>(
             Ok(&field.schema)
         }
         Schema::Array(schema) => Ok(schema),
-        Schema::Union(_) => avro_schema_skip_union(schema),
+        // Only nullable union should be handled here.
+        // We will not extract inner schema for real union (and it's not extractable).
+        Schema::Union(_) => avro_schema_skip_nullable_union(schema),
         Schema::Map(schema) => Ok(schema),
         _ => bail!("avro schema does not have inner item, schema: {:?}", schema),
     }
@@ -476,11 +570,337 @@ pub(crate) fn avro_to_jsonb(avro: &Value, builder: &mut jsonbb::Builder) -> Acce
 mod tests {
     use std::str::FromStr;
 
-    use apache_avro::Decimal as AvroDecimal;
+    use apache_avro::{from_avro_datum, Decimal as AvroDecimal};
+    use expect_test::expect;
     use risingwave_common::types::{Datum, Decimal};
 
     use super::*;
 
+    /// Test the behavior of the Rust Avro lib for handling union with logical type.
+    #[test]
+    fn test_avro_lib_union() {
+        // duplicate types
+        let s = Schema::parse_str(r#"["null", "null"]"#);
+        expect![[r#"
+            Err(
+                Unions cannot contain duplicate types,
+            )
+        "#]]
+        .assert_debug_eq(&s);
+        let s = Schema::parse_str(r#"["int", "int"]"#);
+        expect![[r#"
+            Err(
+                Unions cannot contain duplicate types,
+            )
+        "#]]
+        .assert_debug_eq(&s);
+        // multiple map/array are considered as the same type, regardless of the element type!
+        let s = Schema::parse_str(
+            r#"[
+"null",
+{
+    "type": "map",
+    "values" : "long",
+    "default": {}
+},
+{
+    "type": "map",
+    "values" : "int",
+    "default": {}
+}
+]
+"#,
+        );
+        expect![[r#"
+            Err(
+                Unions cannot contain duplicate types,
+            )
+        "#]]
+        .assert_debug_eq(&s);
+        let s = Schema::parse_str(
+            r#"[
+"null",
+{
+    "type": "array",
+    "items" : "long",
+    "default": {}
+},
+{
+    "type": "array",
+    "items" : "int",
+    "default": {}
+}
+]
+"#,
+        );
+        expect![[r#"
+        Err(
+            Unions cannot contain duplicate types,
+        )
+    "#]]
+        .assert_debug_eq(&s);
+        // multiple named types
+        let s = Schema::parse_str(
+            r#"[
+"null",
+{"type":"fixed","name":"a","size":16},
+{"type":"fixed","name":"b","size":32}
+]
+"#,
+        );
+        expect![[r#"
+            Ok(
+                Union(
+                    UnionSchema {
+                        schemas: [
+                            Null,
+                            Fixed(
+                                FixedSchema {
+                                    name: Name {
+                                        name: "a",
+                                        namespace: None,
+                                    },
+                                    aliases: None,
+                                    doc: None,
+                                    size: 16,
+                                    attributes: {},
+                                },
+                            ),
+                            Fixed(
+                                FixedSchema {
+                                    name: Name {
+                                        name: "b",
+                                        namespace: None,
+                                    },
+                                    aliases: None,
+                                    doc: None,
+                                    size: 32,
+                                    attributes: {},
+                                },
+                            ),
+                        ],
+                        variant_index: {
+                            Null: 0,
+                        },
+                    },
+                ),
+            )
+        "#]]
+        .assert_debug_eq(&s);
+
+        // union in union
+        let s = Schema::parse_str(r#"["int", ["null", "int"]]"#);
+        expect![[r#"
+            Err(
+                Unions may not directly contain a union,
+            )
+        "#]]
+        .assert_debug_eq(&s);
+
+        // logical type
+        let s = Schema::parse_str(r#"["null", {"type":"string","logicalType":"uuid"}]"#).unwrap();
+        expect![[r#"
+            Union(
+                UnionSchema {
+                    schemas: [
+                        Null,
+                        Uuid,
+                    ],
+                    variant_index: {
+                        Null: 0,
+                        Uuid: 1,
+                    },
+                },
+            )
+        "#]]
+        .assert_debug_eq(&s);
+        // Note: Java Avro lib rejects this (logical type unions with its physical type)
+        let s = Schema::parse_str(r#"["string", {"type":"string","logicalType":"uuid"}]"#).unwrap();
+        expect![[r#"
+            Union(
+                UnionSchema {
+                    schemas: [
+                        String,
+                        Uuid,
+                    ],
+                    variant_index: {
+                        String: 0,
+                        Uuid: 1,
+                    },
+                },
+            )
+        "#]]
+        .assert_debug_eq(&s);
+        // Note: Java Avro lib rejects this (logical type unions with its physical type)
+        let s = Schema::parse_str(r#"["int", {"type":"int", "logicalType": "date"}]"#).unwrap();
+        expect![[r#"
+            Union(
+                UnionSchema {
+                    schemas: [
+                        Int,
+                        Date,
+                    ],
+                    variant_index: {
+                        Int: 0,
+                        Date: 1,
+                    },
+                },
+            )
+        "#]]
+        .assert_debug_eq(&s);
+        // Note: Java Avro lib allows this (2 decimal with different "name")
+        let s = Schema::parse_str(
+            r#"[
+{"type":"fixed","name":"Decimal128","size":16,"logicalType":"decimal","precision":38,"scale":2},
+{"type":"fixed","name":"Decimal256","size":32,"logicalType":"decimal","precision":50,"scale":2}
+]"#,
+        );
+        expect![[r#"
+            Err(
+                Unions cannot contain duplicate types,
+            )
+        "#]]
+        .assert_debug_eq(&s);
+    }
+
+    #[test]
+    fn test_avro_lib_union_record_bug() {
+        // multiple named types (record)
+        let s = Schema::parse_str(
+            r#"
+    {
+      "type": "record",
+      "name": "Root",
+      "fields": [
+        {
+          "name": "unionTypeComplex",
+          "type": [
+            "null",
+            {"type": "record", "name": "Email","fields": [{"name":"inner","type":"string"}]},
+            {"type": "record", "name": "Fax","fields": [{"name":"inner","type":"int"}]},
+            {"type": "record", "name": "Sms","fields": [{"name":"inner","type":"int"}]}
+          ]
+        }
+      ]
+    }
+        "#,
+        )
+        .unwrap();
+
+        let bytes = hex::decode("060c").unwrap();
+        // Correct should be variant 3 (Sms)
+        let correct_value = from_avro_datum(&s, &mut bytes.as_slice(), None);
+        expect![[r#"
+                Ok(
+                    Record(
+                        [
+                            (
+                                "unionTypeComplex",
+                                Union(
+                                    3,
+                                    Record(
+                                        [
+                                            (
+                                                "inner",
+                                                Int(
+                                                    6,
+                                                ),
+                                            ),
+                                        ],
+                                    ),
+                                ),
+                            ),
+                        ],
+                    ),
+                )
+            "#]]
+        .assert_debug_eq(&correct_value);
+        // Bug: We got variant 2 (Fax) here, if we pass the reader schema.
+        let wrong_value = from_avro_datum(&s, &mut bytes.as_slice(), Some(&s));
+        expect![[r#"
+                Ok(
+                    Record(
+                        [
+                            (
+                                "unionTypeComplex",
+                                Union(
+                                    2,
+                                    Record(
+                                        [
+                                            (
+                                                "inner",
+                                                Int(
+                                                    6,
+                                                ),
+                                            ),
+                                        ],
+                                    ),
+                                ),
+                            ),
+                        ],
+                    ),
+                )
+            "#]]
+        .assert_debug_eq(&wrong_value);
+
+        // The bug below can explain what happened.
+        // The two records below are actually incompatible: https://avro.apache.org/docs/1.11.1/specification/_print/#schema-resolution
+        // > both schemas are records with the _same (unqualified) name_
+        // In from_avro_datum, it first reads the value with the writer schema, and then
+        // it just uses the reader schema to interpret the value.
+        // The value doesn't have record "name" information. So it wrongly passed the conversion.
+        // The correct way is that we need to use both the writer and reader schema in the second step to interpret the value.
+
+        let s = Schema::parse_str(
+            r#"
+    {
+      "type": "record",
+      "name": "Root",
+      "fields": [
+        {
+          "name": "a",
+          "type": "int"
+        }
+      ]
+    }
+        "#,
+        )
+        .unwrap();
+        let s2 = Schema::parse_str(
+            r#"
+{
+  "type": "record",
+  "name": "Root222",
+  "fields": [
+    {
+      "name": "a",
+      "type": "int"
+    }
+  ]
+}
+    "#,
+        )
+        .unwrap();
+
+        let bytes = hex::decode("0c").unwrap();
+        let value = from_avro_datum(&s, &mut bytes.as_slice(), Some(&s2));
+        expect![[r#"
+            Ok(
+                Record(
+                    [
+                        (
+                            "a",
+                            Int(
+                                6,
+                            ),
+                        ),
+                    ],
+                ),
+            )
+        "#]]
+        .assert_debug_eq(&value);
+    }
+
     #[test]
     fn test_convert_decimal() {
         // 280
diff --git a/src/connector/codec/src/decoder/avro/schema.rs b/src/connector/codec/src/decoder/avro/schema.rs
index 4bd0a469041ad..324b7fd426a56 100644
--- a/src/connector/codec/src/decoder/avro/schema.rs
+++ b/src/connector/codec/src/decoder/avro/schema.rs
@@ -14,14 +14,18 @@
 
 use std::sync::{Arc, LazyLock};
 
+use anyhow::Context;
 use apache_avro::schema::{DecimalSchema, RecordSchema, ResolvedSchema, Schema};
 use apache_avro::AvroResult;
 use itertools::Itertools;
-use risingwave_common::bail;
+use risingwave_common::error::NotImplemented;
 use risingwave_common::log::LogSuppresser;
 use risingwave_common::types::{DataType, Decimal};
+use risingwave_common::{bail, bail_not_implemented};
 use risingwave_pb::plan_common::{AdditionalColumn, ColumnDesc, ColumnDescVersion};
 
+use super::get_nullable_union_inner;
+
 /// Avro schema with `Ref` inlined. The newtype is used to indicate whether the schema is resolved.
 ///
 /// TODO: Actually most of the place should use resolved schema, but currently they just happen to work (Some edge cases are not met yet).
@@ -74,6 +78,8 @@ impl MapHandling {
 
 /// This function expects resolved schema (no `Ref`).
 /// FIXME: require passing resolved schema here.
+/// TODO: change `map_handling` to some `Config`, and also unify debezium.
+/// TODO: use `ColumnDesc` in common instead of PB.
 pub fn avro_schema_to_column_descs(
     schema: &Schema,
     map_handling: Option<MapHandling>,
@@ -196,20 +202,46 @@ fn avro_type_mapping(
             DataType::List(Box::new(item_type))
         }
         Schema::Union(union_schema) => {
-            // We only support using union to represent nullable fields, not general unions.
-            let variants = union_schema.variants();
-            if variants.len() != 2 || !variants.contains(&Schema::Null) {
-                bail!(
-                    "unsupported Avro type, only unions like [null, T] is supported: {:?}",
-                    schema
-                );
-            }
-            let nested_schema = variants
-                .iter()
-                .find_or_first(|s| !matches!(s, Schema::Null))
-                .unwrap();
+            // Note: Unions may not immediately contain other unions. So a `null` must represent a top-level null.
+            // e.g., ["null", ["null", "string"]] is not allowed
+
+            // Note: Unions may not contain more than one schema with the same type, except for the named types record, fixed and enum.
+            // https://avro.apache.org/docs/1.11.1/specification/_print/#unions
+            debug_assert!(
+                union_schema
+                    .variants()
+                    .iter()
+                    .map(Schema::canonical_form) // Schema doesn't implement Eq, but only PartialEq.
+                    .duplicates()
+                    .next()
+                    .is_none(),
+                "Union contains duplicate types: {union_schema:?}",
+            );
+            match get_nullable_union_inner(union_schema) {
+                Some(inner) => avro_type_mapping(inner, map_handling)?,
+                None => {
+                    // Convert the union to a struct, each field of the struct represents a variant of the union.
+                    // Refer to https://github.com/risingwavelabs/risingwave/issues/16273#issuecomment-2179761345 to see why it's not perfect.
+                    // Note: Avro union's variant tag is type name, not field name (unlike Rust enum, or Protobuf oneof).
+
+                    // XXX: do we need to introduce union.handling.mode?
+                    let (fields, field_names) = union_schema
+                        .variants()
+                        .iter()
+                        // null will mean the whole struct is null
+                        .filter(|variant| !matches!(variant, &&Schema::Null))
+                        .map(|variant| {
+                            avro_type_mapping(variant, map_handling).and_then(|t| {
+                                let name = avro_schema_to_struct_field_name(variant)?;
+                                Ok((t, name))
+                            })
+                        })
+                        .process_results(|it| it.unzip::<_, _, Vec<_>, Vec<_>>())
+                        .context("failed to convert Avro union to struct")?;
 
-            avro_type_mapping(nested_schema, map_handling)?
+                    DataType::new_struct(fields, field_names)
+                }
+            }
         }
         Schema::Ref { name } => {
             if name.name == DBZ_VARIABLE_SCALE_DECIMAL_NAME
@@ -217,7 +249,7 @@ fn avro_type_mapping(
             {
                 DataType::Decimal
             } else {
-                bail!("unsupported Avro type: {:?}", schema);
+                bail_not_implemented!("Avro type: {:?}", schema);
             }
         }
         Schema::Map(value_schema) => {
@@ -227,20 +259,25 @@ fn avro_type_mapping(
                     if supported_avro_to_json_type(value_schema) {
                         DataType::Jsonb
                     } else {
-                        bail!(
-                            "unsupported Avro type, cannot convert map to jsonb: {:?}",
+                        bail_not_implemented!(
+                            issue = 16963,
+                            "Avro map type to jsonb: {:?}",
                             schema
-                        )
+                        );
                     }
                 }
                 None => {
+                    // We require it to be specified, because we don't want to have a bad default behavior.
+                    // But perhaps changing the default behavior won't be a breaking change,
+                    // because it affects only on creation time, what the result ColumnDesc will be, and the ColumnDesc will be persisted.
+                    // This is unlike timestamp.handing.mode, which affects parser's behavior on the runtime.
                     bail!("`map.handling.mode` not specified in ENCODE AVRO (...). Currently supported modes: `jsonb`")
                 }
             }
         }
         Schema::Uuid => DataType::Varchar,
         Schema::Null | Schema::Fixed(_) => {
-            bail!("unsupported Avro type: {:?}", schema)
+            bail_not_implemented!("Avro type: {:?}", schema);
         }
     };
 
@@ -278,3 +315,71 @@ fn supported_avro_to_json_type(schema: &Schema) -> bool {
         | Schema::Union(_) => false,
     }
 }
+
+/// The field name when converting Avro union type to RisingWave struct type.
+pub(super) fn avro_schema_to_struct_field_name(schema: &Schema) -> Result<String, NotImplemented> {
+    Ok(match schema {
+        Schema::Null => unreachable!(),
+        Schema::Union(_) => unreachable!(),
+        // Primitive types
+        Schema::Boolean => "boolean".to_string(),
+        Schema::Int => "int".to_string(),
+        Schema::Long => "long".to_string(),
+        Schema::Float => "float".to_string(),
+        Schema::Double => "double".to_string(),
+        Schema::Bytes => "bytes".to_string(),
+        Schema::String => "string".to_string(),
+        // Unnamed Complex types
+        Schema::Array(_) => "array".to_string(),
+        Schema::Map(_) => "map".to_string(),
+        // Named Complex types
+        Schema::Enum(_) | Schema::Ref { name: _ } | Schema::Fixed(_) | Schema::Record(_) => {
+            // schema.name().unwrap().fullname(None)
+            // See test_avro_lib_union_record_bug
+            // https://github.com/risingwavelabs/risingwave/issues/17632
+            bail_not_implemented!(issue=17632, "Avro named type used in Union type: {:?}", schema)
+
+        }
+
+        // Logical types are currently banned. See https://github.com/risingwavelabs/risingwave/issues/17616
+
+/*
+        Schema::Uuid => "uuid".to_string(),
+        // Decimal is the most tricky. https://avro.apache.org/docs/1.11.1/specification/_print/#decimal
+        // - A decimal logical type annotates Avro bytes _or_ fixed types.
+        // - It has attributes `precision` and `scale`.
+        //  "For the purposes of schema resolution, two schemas that are decimal logical types match if their scales and precisions match."
+        // - When the physical type is fixed, it's a named type. And a schema containing 2 decimals is possible:
+        //   [
+        //     {"type":"fixed","name":"Decimal128","size":16,"logicalType":"decimal","precision":38,"scale":2},
+        //     {"type":"fixed","name":"Decimal256","size":32,"logicalType":"decimal","precision":50,"scale":2}
+        //   ]
+        //   In this case (a logical type's physical type is a named type), perhaps we should use the physical type's `name`.
+        Schema::Decimal(_) => "decimal".to_string(),
+        Schema::Date => "date".to_string(),
+        // Note: in Avro, the name style is "time-millis", etc.
+        // But in RisingWave (Postgres), it will require users to use quotes, i.e.,
+        // select (struct)."time-millis", (struct).time_millies from t;
+        // The latter might be more user-friendly.
+        Schema::TimeMillis => "time_millis".to_string(),
+        Schema::TimeMicros => "time_micros".to_string(),
+        Schema::TimestampMillis => "timestamp_millis".to_string(),
+        Schema::TimestampMicros => "timestamp_micros".to_string(),
+        Schema::LocalTimestampMillis => "local_timestamp_millis".to_string(),
+        Schema::LocalTimestampMicros => "local_timestamp_micros".to_string(),
+        Schema::Duration => "duration".to_string(),
+*/
+        Schema::Uuid
+        | Schema::Decimal(_)
+        | Schema::Date
+        | Schema::TimeMillis
+        | Schema::TimeMicros
+        | Schema::TimestampMillis
+        | Schema::TimestampMicros
+        | Schema::LocalTimestampMillis
+        | Schema::LocalTimestampMicros
+        | Schema::Duration => {
+            bail_not_implemented!(issue=17616, "Avro logicalType used in Union type: {:?}", schema)
+        }
+    })
+}
diff --git a/src/connector/codec/src/decoder/mod.rs b/src/connector/codec/src/decoder/mod.rs
index 61d761410fae6..814e06a166c6c 100644
--- a/src/connector/codec/src/decoder/mod.rs
+++ b/src/connector/codec/src/decoder/mod.rs
@@ -16,6 +16,7 @@ pub mod avro;
 pub mod json;
 pub mod utils;
 
+use risingwave_common::error::NotImplemented;
 use risingwave_common::types::{DataType, Datum, DatumCow, ToOwnedDatum};
 use thiserror::Error;
 use thiserror_ext::Macro;
@@ -40,23 +41,44 @@ pub enum AccessError {
     /// Errors that are not categorized into variants above.
     #[error("{message}")]
     Uncategorized { message: String },
+
+    #[error(transparent)]
+    NotImplemented(#[from] NotImplemented),
+    // NOTE: We intentionally don't embed `anyhow::Error` in `AccessError` since it happens
+    // in record-level and it might be too heavy to capture the backtrace
+    // when creating a new `anyhow::Error`.
 }
 
 pub type AccessResult<T = Datum> = std::result::Result<T, AccessError>;
 
-/// Access to a field in the data structure.
+/// Access to a field in the data structure. Created by `AccessBuilder`.
+///
+/// It's the `ENCODE ...` part in `FORMAT ... ENCODE ...`
 pub trait Access {
     /// Accesses `path` in the data structure (*parsed* Avro/JSON/Protobuf data),
     /// and then converts it to RisingWave `Datum`.
+    ///
     /// `type_expected` might or might not be used during the conversion depending on the implementation.
     ///
     /// # Path
     ///
-    /// We usually expect the data is a record (struct), and `path` represents field path.
+    /// We usually expect the data (`Access` instance) is a record (struct), and `path` represents field path.
     /// The data (or part of the data) represents the whole row (`Vec<Datum>`),
     /// and we use different `path` to access one column at a time.
     ///
-    /// e.g., for Avro, we access `["col_name"]`; for Debezium Avro, we access `["before", "col_name"]`.
+    /// TODO: the meaning of `path` is a little confusing and maybe over-abstracted.
+    /// `access` does not need to serve arbitrarily deep `path` access, but just "top-level" access.
+    /// The API creates an illusion that arbitrary access is supported, but it's not.
+    /// Perhapts we should separate out another trait like `ToDatum`,
+    /// which only does type mapping, without caring about the path. And `path` itself is only an `enum` instead of `&[&str]`.
+    ///
+    /// What `path` to access is decided by the CDC layer, i.e., the `FORMAT ...` part (`ChangeEvent`).
+    /// e.g.,
+    /// - `DebeziumChangeEvent` accesses `["before", "col_name"]` for value,
+    ///   `["source", "db"]`, `["source", "table"]` etc. for additional columns' values,
+    ///   `["op"]` for op type.
+    /// - `MaxwellChangeEvent` accesses `["data", "col_name"]` for value, `["type"]` for op type.
+    /// - In the simplest case, for `FORMAT PLAIN/UPSERT` (`KvEvent`), they just access `["col_name"]` for value, and op type is derived.
     ///
     /// # Returns
     ///
diff --git a/src/connector/codec/src/lib.rs b/src/connector/codec/src/lib.rs
index 6b1335ab32e26..cbf0ad14046f7 100644
--- a/src/connector/codec/src/lib.rs
+++ b/src/connector/codec/src/lib.rs
@@ -22,7 +22,6 @@
 #![feature(box_patterns)]
 #![feature(trait_alias)]
 #![feature(lint_reasons)]
-#![feature(lazy_cell)]
 #![feature(let_chains)]
 #![feature(box_into_inner)]
 #![feature(type_alias_impl_trait)]
@@ -44,6 +43,7 @@
 pub mod decoder;
 
 pub use apache_avro::schema::Schema as AvroSchema;
+pub use apache_avro::types::{Value as AvroValue, ValueKind as AvroValueKind};
 pub use risingwave_pb::plan_common::ColumnDesc;
 pub struct JsonSchema(pub serde_json::Value);
 impl JsonSchema {
diff --git a/src/connector/codec/tests/integration_tests/avro.rs b/src/connector/codec/tests/integration_tests/avro.rs
new file mode 100644
index 0000000000000..11421c151d7a5
--- /dev/null
+++ b/src/connector/codec/tests/integration_tests/avro.rs
@@ -0,0 +1,887 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use anyhow::Context;
+use apache_avro::from_avro_datum;
+use risingwave_connector_codec::decoder::avro::{
+    avro_schema_to_column_descs, AvroAccess, AvroParseOptions, MapHandling, ResolvedAvroSchema,
+};
+use risingwave_connector_codec::decoder::Access;
+use risingwave_connector_codec::AvroSchema;
+use thiserror_ext::AsReport;
+
+use crate::utils::*;
+
+/// Refer to `AvroAccessBuilder::parse_avro_value` for how Avro data looks like.
+enum TestDataEncoding {
+    /// Each data is a JSON encoded Avro value.
+    ///
+    /// Refer to: <https://avro.apache.org/docs/1.11.1/specification/#json-encoding>
+    ///
+    /// Specially, it handles `union` variants, and differentiates `bytes` from `string`.
+    ///
+    /// TODO: Not supported yet, because `apache_avro` doesn't support decoding JSON encoded avro..
+    #[allow(dead_code)]
+    Json,
+    /// Each data is a binary encoded Avro value, converted to a hex string.
+    ///
+    /// Tool convert Avro JSON to hex string: <https://xxchan-vercel-playground.vercel.app/avro>
+    HexBinary,
+}
+
+struct Config {
+    /// TODO: For one test data, we can test all possible config options.
+    map_handling: Option<MapHandling>,
+    data_encoding: TestDataEncoding,
+}
+
+fn avro_schema_str_to_risingwave_schema(
+    avro_schema: &str,
+    config: &Config,
+) -> anyhow::Result<(ResolvedAvroSchema, Vec<ColumnDesc>)> {
+    // manually implement some logic in AvroParserConfig::map_to_columns
+    let avro_schema = AvroSchema::parse_str(avro_schema).context("failed to parse Avro schema")?;
+    let resolved_schema =
+        ResolvedAvroSchema::create(avro_schema.into()).context("failed to resolve Avro schema")?;
+
+    let rw_schema =
+        avro_schema_to_column_descs(&resolved_schema.resolved_schema, config.map_handling)
+            .context("failed to convert Avro schema to RisingWave schema")?
+            .iter()
+            .map(ColumnDesc::from)
+            .collect_vec();
+    Ok((resolved_schema, rw_schema))
+}
+
+/// Data driven testing for converting Avro Schema to RisingWave Schema, and then converting Avro data into RisingWave data.
+///
+/// The expected results can be automatically updated. To run and update the tests:
+/// ```bash
+/// UPDATE_EXPECT=1 cargo test -p risingwave_connector_codec
+/// ```
+/// Or use Rust Analyzer. Refer to <https://github.com/rust-analyzer/expect-test>.
+///
+/// ## Arguments
+/// - `avro_schema`: Avro schema in JSON format.
+/// - `avro_data`: list of Avro data. Refer to [`TestDataEncoding`] for the format.
+///
+/// ## Why not directly test the uppermost layer `AvroParserConfig` and `AvroAccessBuilder`?
+///
+/// Because their interface are not clean enough, and have complex logic like schema registry.
+/// We might need to separate logic to make them clenaer and then we can use it directly for testing.
+///
+/// ## If we reimplement a similar logic here, what are we testing?
+///
+/// Basically unit tests of `avro_schema_to_column_descs`, `convert_to_datum`, i.e., the type mapping.
+///
+/// It makes some sense, as the data parsing logic is generally quite simple (one-liner), and the most
+/// complex and error-prone part is the type mapping.
+///
+/// ## Why test schema mapping and data mapping together?
+///
+/// Because the expected data type for data mapping comes from the schema mapping.
+#[track_caller]
+fn check(
+    avro_schema: &str,
+    avro_data: &[&str],
+    config: Config,
+    expected_risingwave_schema: expect_test::Expect,
+    expected_risingwave_data: expect_test::Expect,
+) {
+    let (resolved_schema, rw_schema) =
+        match avro_schema_str_to_risingwave_schema(avro_schema, &config) {
+            Ok(res) => res,
+            Err(e) => {
+                expected_risingwave_schema.assert_eq(&format!("{}", e.as_report()));
+                expected_risingwave_data.assert_eq("");
+                return;
+            }
+        };
+    expected_risingwave_schema.assert_eq(&format!(
+        "{:#?}",
+        rw_schema.iter().map(ColumnDescTestDisplay).collect_vec()
+    ));
+
+    // manually implement some logic in AvroAccessBuilder, and some in PlainParser::parse_inner
+    let mut data_str = vec![];
+    for data in avro_data {
+        let parser = AvroParseOptions::create(&resolved_schema.resolved_schema);
+
+        match config.data_encoding {
+            TestDataEncoding::Json => todo!(),
+            TestDataEncoding::HexBinary => {
+                let data = hex::decode(data).expect("failed to decode hex string");
+                let avro_data =
+                    from_avro_datum(&resolved_schema.original_schema, &mut data.as_slice(), None)
+                        .expect("failed to parse Avro data");
+                let access = AvroAccess::new(&avro_data, parser);
+
+                let mut row = vec![];
+                for col in &rw_schema {
+                    let rw_data = access
+                        .access(&[&col.name], &col.data_type)
+                        .expect("failed to access");
+                    row.push(format!("{:#?}", DatumCowTestDisplay(&rw_data)));
+                }
+                data_str.push(format!("{}", row.iter().format("\n")));
+            }
+        }
+    }
+    expected_risingwave_data.assert_eq(&format!("{}", data_str.iter().format("\n----\n")));
+}
+
+// This corresponds to legacy `scripts/source/test_data/avro_simple_schema_bin.1`. TODO: remove that file.
+#[test]
+fn test_simple() {
+    check(
+        r#"
+{
+  "name": "test_student",
+  "type": "record",
+  "fields": [
+    {
+      "name": "id",
+      "type": "int",
+      "default": 0
+    },
+    {
+      "name": "sequence_id",
+      "type": "long",
+      "default": 0
+    },
+    {
+      "name": "name",
+      "type": ["null", "string"]
+    },
+    {
+      "name": "score",
+      "type": "float",
+      "default": 0.0
+    },
+    {
+      "name": "avg_score",
+      "type": "double",
+      "default": 0.0
+    },
+    {
+      "name": "is_lasted",
+      "type": "boolean",
+      "default": false
+    },
+    {
+      "name": "entrance_date",
+      "type": "int",
+      "logicalType": "date",
+      "default": 0
+    },
+    {
+      "name": "birthday",
+      "type": "long",
+      "logicalType": "timestamp-millis",
+      "default": 0
+    },
+    {
+      "name": "anniversary",
+      "type": "long",
+      "logicalType": "timestamp-micros",
+      "default": 0
+    },
+    {
+      "name": "passed",
+      "type": {
+        "name": "interval",
+        "type": "fixed",
+        "size": 12
+      },
+      "logicalType": "duration"
+    },
+    {
+      "name": "bytes",
+      "type": "bytes",
+      "default": ""
+    }
+  ]
+}
+        "#,
+        &[
+            // {"id":32,"sequence_id":64,"name":{"string":"str_value"},"score":32.0,"avg_score":64.0,"is_lasted":true,"entrance_date":0,"birthday":0,"anniversary":0,"passed":"\u0001\u0000\u0000\u0000\u0001\u0000\u0000\u0000\u00E8\u0003\u0000\u0000","bytes":"\u0001\u0002\u0003\u0004\u0005"}
+              "40800102127374725f76616c7565000000420000000000005040010000000100000001000000e80300000a0102030405"
+            ],
+        Config {
+            map_handling: None,
+            data_encoding: TestDataEncoding::HexBinary,
+        },
+        expect![[r#"
+            [
+                id(#1): Int32,
+                sequence_id(#2): Int64,
+                name(#3): Varchar,
+                score(#4): Float32,
+                avg_score(#5): Float64,
+                is_lasted(#6): Boolean,
+                entrance_date(#7): Date,
+                birthday(#8): Timestamptz,
+                anniversary(#9): Timestamptz,
+                passed(#10): Interval,
+                bytes(#11): Bytea,
+            ]"#]],
+        expect![[r#"
+            Owned(Int32(32))
+            Owned(Int64(64))
+            Borrowed(Utf8("str_value"))
+            Owned(Float32(OrderedFloat(32.0)))
+            Owned(Float64(OrderedFloat(64.0)))
+            Owned(Bool(true))
+            Owned(Date(Date(1970-01-01)))
+            Owned(Timestamptz(Timestamptz(0)))
+            Owned(Timestamptz(Timestamptz(0)))
+            Owned(Interval(Interval { months: 1, days: 1, usecs: 1000000 }))
+            Borrowed(Bytea([1, 2, 3, 4, 5]))"#]],
+    )
+}
+
+#[test]
+fn test_nullable_union() {
+    check(
+        r#"
+{
+  "name": "test_student",
+  "type": "record",
+  "fields": [
+    {
+      "name": "id",
+      "type": "int",
+      "default": 0
+    },
+    {
+      "name": "age",
+      "type": ["null", "int"]
+    },
+    {
+      "name": "sequence_id",
+      "type": ["null", "long"]
+    },
+    {
+      "name": "name",
+      "type": ["null", "string"],
+      "default": null
+    },
+    {
+      "name": "score",
+      "type": [ "float", "null" ],
+      "default": 1.0
+    },
+    {
+      "name": "avg_score",
+      "type": ["null", "double"]
+    },
+    {
+      "name": "is_lasted",
+      "type": ["null", "boolean"]
+    },
+    {
+      "name": "entrance_date",
+      "type": [
+        "null",
+        {
+          "type": "int",
+          "logicalType": "date",
+          "arg.properties": {
+            "range": {
+              "min": 1,
+              "max": 19374
+            }
+          }
+        }
+      ],
+      "default": null
+    },
+    {
+      "name": "birthday",
+      "type": [
+        "null",
+          {
+            "type": "long",
+            "logicalType": "timestamp-millis",
+            "arg.properties": {
+              "range": {
+                "min": 1,
+                "max": 1673970376213
+              }
+            }
+          }
+      ],
+      "default": null
+    },
+    {
+      "name": "anniversary",
+      "type": [
+        "null",
+        {
+          "type" : "long",
+          "logicalType": "timestamp-micros",
+          "arg.properties": {
+            "range": {
+              "min": 1,
+              "max": 1673970376213000
+            }
+          }
+        }
+      ],
+      "default": null
+    }
+  ]
+}
+        "#,
+        &[
+            // {
+            //   "id": 5,
+            //   "age": null,
+            //   "sequence_id": null,
+            //   "name": null,
+            //   "score": null,
+            //   "avg_score": null,
+            //   "is_lasted": null,
+            //   "entrance_date": null,
+            //   "birthday": null,
+            //   "anniversary": null
+            // }
+            "0a000000020000000000",
+        ],
+        Config {
+            map_handling: None,
+            data_encoding: TestDataEncoding::HexBinary,
+        },
+        expect![[r#"
+            [
+                id(#1): Int32,
+                age(#2): Int32,
+                sequence_id(#3): Int64,
+                name(#4): Varchar,
+                score(#5): Float32,
+                avg_score(#6): Float64,
+                is_lasted(#7): Boolean,
+                entrance_date(#8): Date,
+                birthday(#9): Timestamptz,
+                anniversary(#10): Timestamptz,
+            ]"#]],
+        expect![[r#"
+            Owned(Int32(5))
+            Owned(null)
+            Owned(null)
+            Owned(null)
+            Owned(null)
+            Owned(null)
+            Owned(null)
+            Owned(null)
+            Owned(null)
+            Owned(null)"#]],
+    )
+}
+
+/// From `e2e_test/source_inline/kafka/avro/upsert_avro_json`
+#[test]
+fn test_1() {
+    check(
+        r#"
+{
+  "type": "record",
+  "name": "OBJ_ATTRIBUTE_VALUE",
+  "namespace": "CPLM",
+  "fields": [
+    {
+      "name": "op_type",
+      "type": [
+        "null",
+        "string"
+      ],
+      "default": null
+    },
+    {
+      "name": "ID",
+      "type": [
+        "null",
+        "string"
+      ],
+      "default": null
+    },
+    {
+      "name": "CLASS_ID",
+      "type": [
+        "null",
+        "string"
+      ],
+      "default": null
+    },
+    {
+      "name": "ITEM_ID",
+      "type": [
+        "null",
+        "string"
+      ],
+      "default": null
+    },
+    {
+      "name": "ATTR_ID",
+      "type": [
+        "null",
+        "string"
+      ],
+      "default": null
+    },
+    {
+      "name": "ATTR_VALUE",
+      "type": [
+        "null",
+        "string"
+      ],
+      "default": null
+    },
+    {
+      "name": "ORG_ID",
+      "type": [
+        "null",
+        "string"
+      ],
+      "default": null
+    },
+    {
+      "name": "UNIT_INFO",
+      "type": [
+        "null",
+        "string"
+      ],
+      "default": null
+    },
+    {
+      "name": "UPD_TIME",
+      "type": [
+        "null",
+        "string"
+      ],
+      "default": null
+    },
+    {
+      "name": "DEC_VAL",
+      "type": [
+        {
+          "type": "bytes",
+          "logicalType": "decimal",
+          "precision": 10,
+          "scale": 2
+        },
+        "null"
+      ],
+      "default": "\u00ff"
+    },
+    {
+      "name": "REFERRED",
+      "type": [
+        "null",
+        {
+          "type": "record",
+          "name": "REFERRED_TYPE",
+          "fields": [
+            {
+              "name": "a",
+              "type": "string"
+            }
+          ]
+        }
+      ],
+      "default": null
+    },
+    {
+      "name": "REF",
+      "type": [
+        "null",
+        "REFERRED_TYPE"
+      ],
+      "default": null
+    },
+    {
+      "name": "uuid",
+      "type": [
+        "null",
+        {
+          "type": "string",
+          "logicalType": "uuid"
+        }
+      ],
+      "default": null
+    },
+    {
+      "name": "rate",
+      "type": "double",
+      "default": "NaN"
+    }
+  ],
+  "connect.name": "CPLM.OBJ_ATTRIBUTE_VALUE"
+}
+"#,
+        &[
+            // {"op_type": {"string": "update"}, "ID": {"string": "id1"}, "CLASS_ID": {"string": "1"}, "ITEM_ID": {"string": "6768"}, "ATTR_ID": {"string": "6970"}, "ATTR_VALUE": {"string": "value9"}, "ORG_ID": {"string": "7172"}, "UNIT_INFO": {"string": "info9"}, "UPD_TIME": {"string": "2021-05-18T07:59:58.714Z"}, "DEC_VAL": {"bytes": "\u0002\u0054\u000b\u00e3\u00ff"}}
+            "020c7570646174650206696431020231020836373638020836393730020c76616c756539020837313732020a696e666f390230323032312d30352d31385430373a35393a35382e3731345a000a02540be3ff000000000000000000f87f"
+            ],
+        Config {
+            map_handling: None,
+            data_encoding: TestDataEncoding::HexBinary,
+        },
+        expect![[r#"
+            [
+                op_type(#1): Varchar,
+                ID(#2): Varchar,
+                CLASS_ID(#3): Varchar,
+                ITEM_ID(#4): Varchar,
+                ATTR_ID(#5): Varchar,
+                ATTR_VALUE(#6): Varchar,
+                ORG_ID(#7): Varchar,
+                UNIT_INFO(#8): Varchar,
+                UPD_TIME(#9): Varchar,
+                DEC_VAL(#10): Decimal,
+                REFERRED(#11): Struct { a: Varchar },
+                REF(#12): Struct { a: Varchar },
+                uuid(#13): Varchar,
+                rate(#14): Float64,
+            ]"#]],
+        expect![[r#"
+            Borrowed(Utf8("update"))
+            Borrowed(Utf8("id1"))
+            Borrowed(Utf8("1"))
+            Borrowed(Utf8("6768"))
+            Borrowed(Utf8("6970"))
+            Borrowed(Utf8("value9"))
+            Borrowed(Utf8("7172"))
+            Borrowed(Utf8("info9"))
+            Borrowed(Utf8("2021-05-18T07:59:58.714Z"))
+            Owned(Decimal(Normalized(99999999.99)))
+            Owned(null)
+            Owned(null)
+            Owned(null)
+            Owned(Float64(OrderedFloat(NaN)))"#]],
+    );
+}
+
+#[test]
+fn test_union() {
+    // A basic test
+    check(
+        r#"
+{
+  "type": "record",
+  "name": "Root",
+  "fields": [
+    {
+      "name": "unionType",
+      "type": ["int", "string"]
+    },
+    {
+      "name": "unionTypeComplex",
+      "type": [
+        "null",
+        {"type": "record", "name": "Email","fields": [{"name":"inner","type":"string"}]},
+        {"type": "record", "name": "Fax","fields": [{"name":"inner","type":"int"}]},
+        {"type": "record", "name": "Sms","fields": [{"name":"inner","type":"int"}]}
+      ]
+    },
+    {
+      "name": "nullableString",
+      "type": ["null", "string"]
+    }
+  ]
+}
+    "#,
+        &[
+            // {
+            //   "unionType": {"int": 114514},
+            //   "unionTypeComplex": {"Sms": {"inner":6}},
+            //   "nullableString": null
+            // }
+            "00a4fd0d060c00",
+            // {
+            //   "unionType": {"int": 114514},
+            //   "unionTypeComplex": {"Fax": {"inner":6}},
+            //   "nullableString": null
+            // }
+            "00a4fd0d040c00",
+            // {
+            //   "unionType": {"string": "oops"},
+            //   "unionTypeComplex": null,
+            //   "nullableString": {"string": "hello"}
+            // }
+            "02086f6f707300020a68656c6c6f",
+            // {
+            //   "unionType": {"string": "oops"},
+            //   "unionTypeComplex": {"Email": {"inner":"a@b.c"}},
+            //   "nullableString": null
+            // }
+            "02086f6f7073020a6140622e6300",
+        ],
+        Config {
+            map_handling: None,
+            data_encoding: TestDataEncoding::HexBinary,
+        },
+        // FIXME: why the struct type doesn't have field_descs? https://github.com/risingwavelabs/risingwave/issues/17128
+        expect![[r#"
+            failed to convert Avro schema to RisingWave schema: failed to convert Avro union to struct: Feature is not yet implemented: Avro named type used in Union type: Record(RecordSchema { name: Name { name: "Email", namespace: None }, aliases: None, doc: None, fields: [RecordField { name: "inner", doc: None, aliases: None, default: None, schema: String, order: Ascending, position: 0, custom_attributes: {} }], lookup: {"inner": 0}, attributes: {} })
+            Tracking issue: https://github.com/risingwavelabs/risingwave/issues/17632"#]],
+        expect![""],
+    );
+
+    // logicalType is currently rejected
+    // https://github.com/risingwavelabs/risingwave/issues/17616
+    check(
+        r#"
+{
+"type": "record",
+"name": "Root",
+"fields": [
+  {
+    "name": "unionLogical",
+    "type": ["int", {"type":"int", "logicalType": "date"}]
+  }
+]
+}
+  "#,
+        &[],
+        Config {
+            map_handling: None,
+            data_encoding: TestDataEncoding::HexBinary,
+        },
+        expect![[r#"
+            failed to convert Avro schema to RisingWave schema: failed to convert Avro union to struct: Feature is not yet implemented: Avro logicalType used in Union type: Date
+            Tracking issue: https://github.com/risingwavelabs/risingwave/issues/17616"#]],
+        expect![""],
+    );
+
+    // test named type. Consider namespace.
+    // https://avro.apache.org/docs/1.11.1/specification/_print/#names
+    // List of things to take care:
+    // - Record fields and enum symbols DO NOT have namespace.
+    // - If the name specified contains a dot, then it is assumed to be a fullname, and any namespace also specified is IGNORED.
+    // - If a name doesn't have its own namespace, it will look for its most tightly enclosing named schema.
+    check(
+        r#"
+{
+    "type": "record",
+    "name": "Root",
+    "namespace": "RootNamespace",
+    "fields": [
+        {
+            "name": "littleFieldToMakeNestingLooksBetter",
+            "type": ["null","int"], "default": null
+        },
+        {
+            "name": "recordField",
+            "type": ["null", "int", {
+                "type": "record",
+                "name": "my.name.spaced.record",
+                "namespace": "when.name.contains.dot.namespace.is.ignored",
+                "fields": [
+                    {"name": "hello", "type": {"type": "int", "default": 1}},
+                    {"name": "world", "type": {"type": "double", "default": 1}}
+                ]
+            }],
+            "default": null
+        },
+        {
+            "name": "enumField",
+            "type": ["null", "int", {
+                "type": "enum",
+                "name": "myEnum",
+                "namespace": "my.namespace",
+                "symbols": ["A", "B", "C", "D"]
+            }],
+            "default": null
+        },
+        {
+            "name": "anotherEnumFieldUsingRootNamespace",
+            "type": ["null", "int", {
+                "type": "enum",
+                "name": "myEnum",
+                "symbols": ["A", "B", "C", "D"]
+            }],
+            "default": null
+        }
+    ]
+}
+"#,
+        &[
+            // {
+            //   "enumField":{"my.namespace.myEnum":"A"},
+            //   "anotherEnumFieldUsingRootNamespace":{"RootNamespace.myEnum": "D"}
+            // }
+            "000004000406",
+        ],
+        Config {
+            map_handling: None,
+            data_encoding: TestDataEncoding::HexBinary,
+        },
+        expect![[r#"
+            failed to convert Avro schema to RisingWave schema: failed to convert Avro union to struct: Feature is not yet implemented: Avro named type used in Union type: Record(RecordSchema { name: Name { name: "record", namespace: Some("my.name.spaced") }, aliases: None, doc: None, fields: [RecordField { name: "hello", doc: None, aliases: None, default: None, schema: Int, order: Ascending, position: 0, custom_attributes: {} }, RecordField { name: "world", doc: None, aliases: None, default: None, schema: Double, order: Ascending, position: 1, custom_attributes: {} }], lookup: {"hello": 0, "world": 1}, attributes: {} })
+            Tracking issue: https://github.com/risingwavelabs/risingwave/issues/17632"#]],
+        expect![""],
+    );
+
+    // This is provided by a user https://github.com/risingwavelabs/risingwave/issues/16273#issuecomment-2051480710
+    check(
+        r#"
+{
+  "namespace": "com.abc.efg.mqtt",
+  "name": "also.DataMessage",
+  "type": "record",
+  "fields": [
+      {
+          "name": "metrics",
+          "type": {
+              "type": "array",
+              "items": {
+                  "name": "also_data_metric",
+                  "type": "record",
+                  "fields": [
+                      {
+                          "name": "id",
+                          "type": "string"
+                      },
+                      {
+                          "name": "name",
+                          "type": "string"
+                      },
+                      {
+                          "name": "norm_name",
+                          "type": [
+                              "null",
+                              "string"
+                          ],
+                          "default": null
+                      },
+                      {
+                          "name": "uom",
+                          "type": [
+                              "null",
+                              "string"
+                          ],
+                          "default": null
+                      },
+                      {
+                          "name": "data",
+                          "type": {
+                              "type": "array",
+                              "items": {
+                                  "name": "dataItem",
+                                  "type": "record",
+                                  "fields": [
+                                      {
+                                          "name": "ts",
+                                          "type": "string",
+                                          "doc": "Timestamp of the metric."
+                                      },
+                                      {
+                                          "name": "value",
+                                          "type": [
+                                              "null",
+                                              "boolean",
+                                              "double",
+                                              "string"
+                                          ],
+                                          "doc": "Value of the metric."
+                                      }
+                                  ]
+                              }
+                          },
+                          "doc": "The data message"
+                      }
+                  ],
+                  "doc": "A metric object"
+              }
+          },
+          "doc": "A list of metrics."
+      }
+  ]
+}
+          "#,
+        &[
+            // {
+            //   "metrics": [
+            //     {"id":"foo", "name":"a", "data": [] }
+            //   ]
+            // }
+            "0206666f6f026100000000",
+            // {
+            //   "metrics": [
+            //     {"id":"foo", "name":"a", "norm_name": null, "uom": {"string":"c"}, "data": [{"ts":"1", "value":null}, {"ts":"2", "value": {"boolean": true }}] }
+            //   ]
+            // }
+            "0206666f6f02610002026304023100023202010000",
+        ],
+        Config {
+            map_handling: None,
+            data_encoding: TestDataEncoding::HexBinary,
+        },
+        expect![[r#"
+            [
+                metrics(#1): List(
+                    Struct {
+                        id: Varchar,
+                        name: Varchar,
+                        norm_name: Varchar,
+                        uom: Varchar,
+                        data: List(
+                            Struct {
+                                ts: Varchar,
+                                value: Struct {
+                                    boolean: Boolean,
+                                    double: Float64,
+                                    string: Varchar,
+                                },
+                            },
+                        ),
+                    },
+                ),
+            ]"#]],
+        expect![[r#"
+            Owned([
+                StructValue(
+                    Utf8("foo"),
+                    Utf8("a"),
+                    null,
+                    null,
+                    [],
+                ),
+            ])
+            ----
+            Owned([
+                StructValue(
+                    Utf8("foo"),
+                    Utf8("a"),
+                    null,
+                    Utf8("c"),
+                    [
+                        StructValue(
+                            Utf8("1"),
+                            null,
+                        ),
+                        StructValue(
+                            Utf8("2"),
+                            StructValue(
+                                Bool(true),
+                                null,
+                                null,
+                            ),
+                        ),
+                    ],
+                ),
+            ])"#]],
+    );
+}
diff --git a/src/stream/src/common/cache/mod.rs b/src/connector/codec/tests/integration_tests/main.rs
similarity index 87%
rename from src/stream/src/common/cache/mod.rs
rename to src/connector/codec/tests/integration_tests/main.rs
index 27481541492e0..8c718f918d0a6 100644
--- a/src/stream/src/common/cache/mod.rs
+++ b/src/connector/codec/tests/integration_tests/main.rs
@@ -12,8 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-mod state_cache;
-mod top_n_cache;
+mod avro;
 
-pub use state_cache::*;
-pub use top_n_cache::*;
+pub mod utils;
diff --git a/src/connector/codec/tests/integration_tests/utils.rs b/src/connector/codec/tests/integration_tests/utils.rs
new file mode 100644
index 0000000000000..cecb0796c455a
--- /dev/null
+++ b/src/connector/codec/tests/integration_tests/utils.rs
@@ -0,0 +1,182 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub use expect_test::{expect, Expect};
+pub use itertools::Itertools;
+pub use risingwave_common::catalog::ColumnDesc;
+use risingwave_common::types::{
+    DataType, Datum, DatumCow, DatumRef, ScalarImpl, ScalarRefImpl, ToDatumRef,
+};
+use risingwave_pb::plan_common::AdditionalColumn;
+
+/// More concise display for `DataType`, to use in tests.
+pub struct DataTypeTestDisplay<'a>(pub &'a DataType);
+
+impl<'a> std::fmt::Debug for DataTypeTestDisplay<'a> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self.0 {
+            DataType::Struct(s) => {
+                if s.len() == 1 {
+                    // avoid multiline display for single field struct
+                    let (name, ty) = s.iter().next().unwrap();
+                    return write!(f, "Struct {{ {}: {:?} }}", name, &DataTypeTestDisplay(ty));
+                }
+
+                let mut f = f.debug_struct("Struct");
+                for (name, ty) in s.iter() {
+                    f.field(name, &DataTypeTestDisplay(ty));
+                }
+                f.finish()?;
+                Ok(())
+            }
+            DataType::List(t) => f
+                .debug_tuple("List")
+                .field(&DataTypeTestDisplay(t))
+                .finish(),
+            _ => {
+                // do not use alternative display for simple types
+                write!(f, "{:?}", self.0)
+            }
+        }
+    }
+}
+
+/// More concise display for `ScalarRefImpl`, to use in tests.
+pub struct ScalarRefImplTestDisplay<'a>(pub ScalarRefImpl<'a>);
+
+impl<'a> std::fmt::Debug for ScalarRefImplTestDisplay<'a> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self.0 {
+            ScalarRefImpl::Struct(s) => {
+                if s.iter_fields_ref().len() == 1 {
+                    // avoid multiline display for single field struct
+                    let field = s.iter_fields_ref().next().unwrap();
+                    return write!(f, "StructValue({:#?})", &DatumRefTestDisplay(field));
+                }
+
+                let mut f = f.debug_tuple("StructValue");
+                for field in s.iter_fields_ref() {
+                    f.field(&DatumRefTestDisplay(field));
+                }
+                f.finish()?;
+                Ok(())
+            }
+            ScalarRefImpl::List(l) => f
+                .debug_list()
+                .entries(l.iter().map(DatumRefTestDisplay))
+                .finish(),
+            _ => {
+                // do not use alternative display for simple types
+                write!(f, "{:?}", self.0)
+            }
+        }
+    }
+}
+
+/// More concise display for `ScalarImpl`, to use in tests.
+pub struct ScalarImplTestDisplay<'a>(pub &'a ScalarImpl);
+
+impl<'a> std::fmt::Debug for ScalarImplTestDisplay<'a> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        ScalarRefImplTestDisplay(self.0.as_scalar_ref_impl()).fmt(f)
+    }
+}
+
+/// More concise display for `DatumRef`, to use in tests.
+pub struct DatumRefTestDisplay<'a>(pub DatumRef<'a>);
+
+impl<'a> std::fmt::Debug for DatumRefTestDisplay<'a> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self.0 {
+            Some(scalar) => ScalarRefImplTestDisplay(scalar).fmt(f),
+            None => write!(f, "null"),
+        }
+    }
+}
+
+/// More concise display for `Datum`, to use in tests.
+pub struct DatumTestDisplay<'a>(pub &'a Datum);
+
+impl<'a> std::fmt::Debug for DatumTestDisplay<'a> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        DatumRefTestDisplay(self.0.to_datum_ref()).fmt(f)
+    }
+}
+
+/// More concise display for `DatumCow`, to use in tests.
+pub struct DatumCowTestDisplay<'a>(pub &'a DatumCow<'a>);
+
+impl<'a> std::fmt::Debug for DatumCowTestDisplay<'a> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self.0 {
+            DatumCow::Borrowed(datum_ref) => {
+                // don't use debug_tuple to avoid extra newline
+                write!(f, "Borrowed(")?;
+                DatumRefTestDisplay(*datum_ref).fmt(f)?;
+                write!(f, ")")?;
+            }
+            DatumCow::Owned(datum) => {
+                write!(f, "Owned(")?;
+                DatumTestDisplay(datum).fmt(f)?;
+                write!(f, ")")?;
+            }
+        }
+        Ok(())
+    }
+}
+
+/// More concise display for `ColumnDesc`, to use in tests.
+pub struct ColumnDescTestDisplay<'a>(pub &'a ColumnDesc);
+
+impl<'a> std::fmt::Debug for ColumnDescTestDisplay<'a> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let ColumnDesc {
+            data_type,
+            column_id,
+            name,
+            field_descs,
+            type_name,
+            generated_or_default_column,
+            description,
+            additional_column: AdditionalColumn { column_type },
+            version: _,
+        } = &self.0;
+
+        write!(
+            f,
+            "{name}(#{column_id}): {:#?}",
+            DataTypeTestDisplay(data_type)
+        )?;
+        if !type_name.is_empty() {
+            write!(f, ", type_name: {}", type_name)?;
+        }
+        if !field_descs.is_empty() {
+            write!(f, ", field_descs: {:?}", field_descs)?;
+        }
+        if let Some(generated_or_default_column) = generated_or_default_column {
+            write!(
+                f,
+                ", generated_or_default_column: {:?}",
+                generated_or_default_column
+            )?;
+        }
+        if let Some(description) = description {
+            write!(f, ", description: {:?}", description)?;
+        }
+        if let Some(column_type) = column_type {
+            write!(f, ", additional_column: {:?}", column_type)?;
+        }
+        Ok(())
+    }
+}
diff --git a/src/connector/src/connector_common/common.rs b/src/connector/src/connector_common/common.rs
index 92bb7d9c30677..0933f38dc14e0 100644
--- a/src/connector/src/connector_common/common.rs
+++ b/src/connector/src/connector_common/common.rs
@@ -601,7 +601,7 @@ impl NatsCommon {
             }
             "plain" => {}
             _ => {
-                bail!("nats connect mode only accept user_and_password/credential/plain");
+                bail!("nats connect mode only accepts user_and_password/credential/plain");
             }
         };
 
diff --git a/src/connector/src/error.rs b/src/connector/src/error.rs
index 3a86062d18a07..9dbd17ae58a1d 100644
--- a/src/connector/src/error.rs
+++ b/src/connector/src/error.rs
@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use risingwave_common::array::ArrayError;
 use risingwave_common::error::def_anyhow_newtype;
 use risingwave_pb::PbFieldNotFound;
 use risingwave_rpc_client::error::RpcError;
@@ -41,17 +42,21 @@ def_anyhow_newtype! {
     url::ParseError => "failed to parse url",
     serde_json::Error => "failed to parse json",
     csv::Error => "failed to parse csv",
+
     uuid::Error => transparent, // believed to be self-explanatory
 
     // Connector errors
     opendal::Error => transparent, // believed to be self-explanatory
-
+    parquet::errors::ParquetError => transparent,
+    ArrayError => "Array error",
     sqlx::Error => transparent, // believed to be self-explanatory
     mysql_async::Error => "MySQL error",
     tokio_postgres::Error => "Postgres error",
+    tiberius::error::Error => "Sql Server error",
     apache_avro::Error => "Avro error",
     rdkafka::error::KafkaError => "Kafka error",
     pulsar::Error => "Pulsar error",
+
     async_nats::jetstream::consumer::StreamError => "Nats error",
     async_nats::jetstream::consumer::pull::MessagesError => "Nats error",
     async_nats::jetstream::context::CreateStreamError => "Nats error",
@@ -68,6 +73,7 @@ def_anyhow_newtype! {
     mongodb::error::Error => "Mongodb error",
 
     openssl::error::ErrorStack => "OpenSSL error",
+    risingwave_common::secret::SecretError => "Secret error",
 }
 
 pub type ConnectorResult<T, E = ConnectorError> = std::result::Result<T, E>;
diff --git a/src/connector/src/lib.rs b/src/connector/src/lib.rs
index 8c0ade401cb47..02a3b8c84b50f 100644
--- a/src/connector/src/lib.rs
+++ b/src/connector/src/lib.rs
@@ -20,7 +20,6 @@
 #![feature(box_patterns)]
 #![feature(trait_alias)]
 #![feature(lint_reasons)]
-#![feature(lazy_cell)]
 #![feature(let_chains)]
 #![feature(box_into_inner)]
 #![feature(type_alias_impl_trait)]
@@ -58,7 +57,7 @@ pub use paste::paste;
 pub use risingwave_jni_core::{call_method, call_static_method, jvm_runtime};
 
 mod with_options;
-pub use with_options::WithPropertiesExt;
+pub use with_options::{WithOptionsSecResolved, WithPropertiesExt};
 
 #[cfg(test)]
 mod with_options_test;
diff --git a/src/connector/src/macros.rs b/src/connector/src/macros.rs
index 48f4e16c747b1..b422a029fa761 100644
--- a/src/connector/src/macros.rs
+++ b/src/connector/src/macros.rs
@@ -21,7 +21,8 @@ macro_rules! for_all_classified_sources {
                 { Mysql },
                 { Postgres },
                 { Citus },
-                { Mongodb }
+                { Mongodb },
+                { SqlServer }
             },
             // other sources
             // todo: file source do not nest with mq source.
diff --git a/src/connector/src/parser/avro/schema_resolver.rs b/src/connector/src/parser/avro/confluent_resolver.rs
similarity index 100%
rename from src/connector/src/parser/avro/schema_resolver.rs
rename to src/connector/src/parser/avro/confluent_resolver.rs
diff --git a/src/connector/src/parser/avro/glue_resolver.rs b/src/connector/src/parser/avro/glue_resolver.rs
new file mode 100644
index 0000000000000..fe0188161ab2d
--- /dev/null
+++ b/src/connector/src/parser/avro/glue_resolver.rs
@@ -0,0 +1,220 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::HashMap;
+use std::sync::Arc;
+
+use anyhow::Context;
+use apache_avro::Schema;
+use aws_sdk_glue::types::{SchemaId, SchemaVersionNumber};
+use aws_sdk_glue::Client;
+use moka::future::Cache;
+
+use crate::connector_common::AwsAuthProps;
+use crate::error::ConnectorResult;
+
+/// Fetch schemas from AWS Glue schema registry and cache them.
+///
+/// Background: This is mainly used for Avro **writer schema** (during schema evolution): When decoding an Avro message,
+/// we must get the message's schema id, and use the *exactly same schema* to decode the message, and then
+/// convert it with the reader schema. (This is also why Avro has to be used with a schema registry instead of a static schema file.)
+///
+/// TODO: support protobuf (not sure if it's needed)
+pub trait GlueSchemaCache {
+    /// Gets the a specific schema by id, which is used as *writer schema*.
+    async fn get_by_id(&self, schema_version_id: uuid::Uuid) -> ConnectorResult<Arc<Schema>>;
+    /// Gets the latest schema by arn, which is used as *reader schema*.
+    async fn get_by_name(&self, schema_arn: &str) -> ConnectorResult<Arc<Schema>>;
+}
+
+#[derive(Debug)]
+pub enum GlueSchemaCacheImpl {
+    Real(RealGlueSchemaCache),
+    Mock(MockGlueSchemaCache),
+}
+
+impl GlueSchemaCacheImpl {
+    pub async fn new(
+        aws_auth_props: &AwsAuthProps,
+        mock_config: Option<&str>,
+    ) -> ConnectorResult<Self> {
+        if let Some(mock_config) = mock_config {
+            return Ok(Self::Mock(MockGlueSchemaCache::new(mock_config)));
+        }
+        Ok(Self::Real(RealGlueSchemaCache::new(aws_auth_props).await?))
+    }
+}
+
+impl GlueSchemaCache for GlueSchemaCacheImpl {
+    async fn get_by_id(&self, schema_version_id: uuid::Uuid) -> ConnectorResult<Arc<Schema>> {
+        match self {
+            Self::Real(inner) => inner.get_by_id(schema_version_id).await,
+            Self::Mock(inner) => inner.get_by_id(schema_version_id).await,
+        }
+    }
+
+    async fn get_by_name(&self, schema_arn: &str) -> ConnectorResult<Arc<Schema>> {
+        match self {
+            Self::Real(inner) => inner.get_by_name(schema_arn).await,
+            Self::Mock(inner) => inner.get_by_name(schema_arn).await,
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct RealGlueSchemaCache {
+    writer_schemas: Cache<uuid::Uuid, Arc<Schema>>,
+    glue_client: Client,
+}
+
+impl RealGlueSchemaCache {
+    /// Create a new `GlueSchemaCache`
+    pub async fn new(aws_auth_props: &AwsAuthProps) -> ConnectorResult<Self> {
+        let client = Client::new(&aws_auth_props.build_config().await?);
+        Ok(Self {
+            writer_schemas: Cache::new(u64::MAX),
+            glue_client: client,
+        })
+    }
+
+    async fn parse_and_cache_schema(
+        &self,
+        schema_version_id: uuid::Uuid,
+        content: &str,
+    ) -> ConnectorResult<Arc<Schema>> {
+        let schema = Schema::parse_str(content).context("failed to parse avro schema")?;
+        let schema = Arc::new(schema);
+        self.writer_schemas
+            .insert(schema_version_id, Arc::clone(&schema))
+            .await;
+        Ok(schema)
+    }
+}
+
+impl GlueSchemaCache for RealGlueSchemaCache {
+    /// Gets the a specific schema by id, which is used as *writer schema*.
+    async fn get_by_id(&self, schema_version_id: uuid::Uuid) -> ConnectorResult<Arc<Schema>> {
+        if let Some(schema) = self.writer_schemas.get(&schema_version_id).await {
+            return Ok(schema);
+        }
+        let res = self
+            .glue_client
+            .get_schema_version()
+            .schema_version_id(schema_version_id)
+            .send()
+            .await
+            .context("glue sdk error")?;
+        let definition = res
+            .schema_definition()
+            .context("glue sdk response without definition")?;
+        self.parse_and_cache_schema(schema_version_id, definition)
+            .await
+    }
+
+    /// Gets the latest schema by arn, which is used as *reader schema*.
+    async fn get_by_name(&self, schema_arn: &str) -> ConnectorResult<Arc<Schema>> {
+        let res = self
+            .glue_client
+            .get_schema_version()
+            .schema_id(SchemaId::builder().schema_arn(schema_arn).build())
+            .schema_version_number(SchemaVersionNumber::builder().latest_version(true).build())
+            .send()
+            .await
+            .context("glue sdk error")?;
+        let schema_version_id = res
+            .schema_version_id()
+            .context("glue sdk response without schema version id")?
+            .parse()
+            .context("glue sdk response invalid schema version id")?;
+        let definition = res
+            .schema_definition()
+            .context("glue sdk response without definition")?;
+        self.parse_and_cache_schema(schema_version_id, definition)
+            .await
+    }
+}
+
+#[derive(Debug)]
+pub struct MockGlueSchemaCache {
+    by_id: HashMap<uuid::Uuid, Arc<Schema>>,
+    arn_to_latest_id: HashMap<String, uuid::Uuid>,
+}
+
+impl MockGlueSchemaCache {
+    pub fn new(mock_config: &str) -> Self {
+        // The `mock_config` accepted is a JSON that looks like:
+        // {
+        //   "by_id": {
+        //     "4dc80ccf-2d0c-4846-9325-7e1c9e928121": {
+        //       "type": "record",
+        //       "name": "MyEvent",
+        //       "fields": [...]
+        //     },
+        //     "3df022f4-b16d-4afe-bdf7-cf4baf8d01d3": {
+        //       ...
+        //     }
+        //   },
+        //   "arn_to_latest_id": {
+        //     "arn:aws:glue:ap-southeast-1:123456123456:schema/default-registry/MyEvent": "3df022f4-b16d-4afe-bdf7-cf4baf8d01d3"
+        //   }
+        // }
+        //
+        // The format is not public and we can make breaking changes to it.
+        // Current format only supports avsc.
+        let parsed: serde_json::Value =
+            serde_json::from_str(mock_config).expect("mock config shall be valid json");
+        let by_id = parsed
+            .get("by_id")
+            .unwrap()
+            .as_object()
+            .unwrap()
+            .iter()
+            .map(|(schema_version_id, schema)| {
+                let schema_version_id = schema_version_id.parse().unwrap();
+                let schema = Schema::parse(schema).unwrap();
+                (schema_version_id, Arc::new(schema))
+            })
+            .collect();
+        let arn_to_latest_id = parsed
+            .get("arn_to_latest_id")
+            .unwrap()
+            .as_object()
+            .unwrap()
+            .iter()
+            .map(|(arn, latest_id)| (arn.clone(), latest_id.as_str().unwrap().parse().unwrap()))
+            .collect();
+        Self {
+            by_id,
+            arn_to_latest_id,
+        }
+    }
+}
+
+impl GlueSchemaCache for MockGlueSchemaCache {
+    async fn get_by_id(&self, schema_version_id: uuid::Uuid) -> ConnectorResult<Arc<Schema>> {
+        Ok(self
+            .by_id
+            .get(&schema_version_id)
+            .context("schema version id not found in mock registry")?
+            .clone())
+    }
+
+    async fn get_by_name(&self, schema_arn: &str) -> ConnectorResult<Arc<Schema>> {
+        let schema_version_id = self
+            .arn_to_latest_id
+            .get(schema_arn)
+            .context("schema arn not found in mock registry")?;
+        self.get_by_id(*schema_version_id).await
+    }
+}
diff --git a/src/connector/src/parser/avro/mod.rs b/src/connector/src/parser/avro/mod.rs
index 19193035bd567..536c700efef8e 100644
--- a/src/connector/src/parser/avro/mod.rs
+++ b/src/connector/src/parser/avro/mod.rs
@@ -12,8 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+mod confluent_resolver;
+mod glue_resolver;
 mod parser;
-mod schema_resolver;
 
+pub use confluent_resolver::ConfluentSchemaCache;
+pub use glue_resolver::{GlueSchemaCache, GlueSchemaCacheImpl};
 pub use parser::{AvroAccessBuilder, AvroParserConfig};
-pub use schema_resolver::ConfluentSchemaCache;
diff --git a/src/connector/src/parser/avro/parser.rs b/src/connector/src/parser/avro/parser.rs
index f6abe86a17f0d..ac93ab3e69807 100644
--- a/src/connector/src/parser/avro/parser.rs
+++ b/src/connector/src/parser/avro/parser.rs
@@ -24,11 +24,13 @@ use risingwave_connector_codec::decoder::avro::{
 };
 use risingwave_pb::plan_common::ColumnDesc;
 
-use super::ConfluentSchemaCache;
+use super::{ConfluentSchemaCache, GlueSchemaCache as _, GlueSchemaCacheImpl};
 use crate::error::ConnectorResult;
 use crate::parser::unified::AccessImpl;
 use crate::parser::util::bytes_from_url;
-use crate::parser::{AccessBuilder, AvroProperties, EncodingProperties, EncodingType, MapHandling};
+use crate::parser::{
+    AccessBuilder, AvroProperties, EncodingProperties, MapHandling, SchemaLocation,
+};
 use crate::schema::schema_registry::{
     extract_schema_id, get_subject_by_strategy, handle_sr_list, Client,
 };
@@ -38,7 +40,7 @@ use crate::schema::schema_registry::{
 pub struct AvroAccessBuilder {
     schema: Arc<ResolvedAvroSchema>,
     /// Refer to [`AvroParserConfig::writer_schema_cache`].
-    pub writer_schema_cache: Option<Arc<ConfluentSchemaCache>>,
+    writer_schema_cache: WriterSchemaCache,
     value: Option<Value>,
 }
 
@@ -53,18 +55,14 @@ impl AccessBuilder for AvroAccessBuilder {
 }
 
 impl AvroAccessBuilder {
-    pub fn new(config: AvroParserConfig, encoding_type: EncodingType) -> ConnectorResult<Self> {
+    pub fn new(config: AvroParserConfig) -> ConnectorResult<Self> {
         let AvroParserConfig {
             schema,
-            key_schema,
             writer_schema_cache,
             ..
         } = config;
         Ok(Self {
-            schema: match encoding_type {
-                EncodingType::Key => key_schema.context("Avro with empty key schema")?,
-                EncodingType::Value => schema,
-            },
+            schema,
             writer_schema_cache,
             value: None,
         })
@@ -72,23 +70,72 @@ impl AvroAccessBuilder {
 
     /// Note: we should use unresolved schema to parsing bytes into avro value.
     /// Otherwise it's an invalid schema and parsing will fail. (Avro error: Two named schema defined for same fullname)
+    ///
+    /// # Notes about how Avro data looks like
+    ///
+    /// First, it has two [serialization encodings: binary and JSON](https://avro.apache.org/docs/1.11.1/specification/#encodings).
+    /// They don't have magic bytes and cannot be distinguished on their own.
+    ///
+    /// But in different cases, it starts with different headers, or magic bytes, which can be confusing.
+    ///
+    /// ## `apache_avro` API and headers
+    ///
+    /// - `apache_avro::Reader`: [Object Container Files](https://avro.apache.org/docs/1.11.1/specification/#object-container-files): contains file header, starting with 4 bytes `Obj1`. This is a batch file encoding. We don't use it.
+    /// - `apache_avro::GenericSingleObjectReader`: [Single-object encoding](https://avro.apache.org/docs/1.11.1/specification/#single-object-encoding): starts with 2 bytes `0xC301`. This is designed to be used in places like Kafka, but Confluent schema registry doesn't use it.
+    /// - `apache_avro::from_avro_datum`: no header, binary encoding. This is what we should use.
+    ///
+    /// ## Confluent schema registry
+    ///
+    /// - In Kafka ([Confluent schema registry wire format](https://docs.confluent.io/platform/7.6/schema-registry/fundamentals/serdes-develop/index.html#wire-format)):
+    ///   starts with 5 bytes`0x00{schema_id:08x}` followed by Avro binary encoding.
     async fn parse_avro_value(&self, payload: &[u8]) -> ConnectorResult<Option<Value>> {
         // parse payload to avro value
         // if use confluent schema, get writer schema from confluent schema registry
-        if let Some(resolver) = &self.writer_schema_cache {
-            let (schema_id, mut raw_payload) = extract_schema_id(payload)?;
-            let writer_schema = resolver.get_by_id(schema_id).await?;
-            Ok(Some(from_avro_datum(
-                writer_schema.as_ref(),
-                &mut raw_payload,
-                Some(&self.schema.original_schema),
-            )?))
-        } else {
-            let mut reader = Reader::with_schema(&self.schema.original_schema, payload)?;
-            match reader.next() {
-                Some(Ok(v)) => Ok(Some(v)),
-                Some(Err(e)) => Err(e)?,
-                None => bail!("avro parse unexpected eof"),
+        match &self.writer_schema_cache {
+            WriterSchemaCache::Confluent(resolver) => {
+                let (schema_id, mut raw_payload) = extract_schema_id(payload)?;
+                let writer_schema = resolver.get_by_id(schema_id).await?;
+                Ok(Some(from_avro_datum(
+                    writer_schema.as_ref(),
+                    &mut raw_payload,
+                    Some(&self.schema.original_schema),
+                )?))
+            }
+            WriterSchemaCache::File => {
+                // FIXME: we should not use `Reader` (file header) here. See comment above and https://github.com/risingwavelabs/risingwave/issues/12871
+                let mut reader = Reader::with_schema(&self.schema.original_schema, payload)?;
+                match reader.next() {
+                    Some(Ok(v)) => Ok(Some(v)),
+                    Some(Err(e)) => Err(e)?,
+                    None => bail!("avro parse unexpected eof"),
+                }
+            }
+            WriterSchemaCache::Glue(resolver) => {
+                // <https://github.com/awslabs/aws-glue-schema-registry/blob/v1.1.20/common/src/main/java/com/amazonaws/services/schemaregistry/utils/AWSSchemaRegistryConstants.java#L59-L61>
+                // byte 0:      header version = 3
+                // byte 1:      compression: 0 = no compression; 5 = zlib (unsupported)
+                // byte 2..=17: 16-byte UUID as schema version id
+                // byte 18..:   raw avro payload
+                if payload.len() < 18 {
+                    bail!("payload shorter than 18-byte glue header");
+                }
+                if payload[0] != 3 {
+                    bail!(
+                        "Only support glue header version 3 but found {}",
+                        payload[0]
+                    );
+                }
+                if payload[1] != 0 {
+                    bail!("Non-zero compression {} not supported", payload[1]);
+                }
+                let schema_version_id = uuid::Uuid::from_slice(&payload[2..18]).unwrap();
+                let writer_schema = resolver.get_by_id(schema_version_id).await?;
+                let mut raw_payload = &payload[18..];
+                Ok(Some(from_avro_datum(
+                    writer_schema.as_ref(),
+                    &mut raw_payload,
+                    Some(&self.schema.original_schema),
+                )?))
             }
         }
     }
@@ -96,83 +143,88 @@ impl AvroAccessBuilder {
 
 #[derive(Debug, Clone)]
 pub struct AvroParserConfig {
-    pub schema: Arc<ResolvedAvroSchema>,
-    pub key_schema: Option<Arc<ResolvedAvroSchema>>,
+    schema: Arc<ResolvedAvroSchema>,
     /// Writer schema is the schema used to write the data. When parsing Avro data, the exactly same schema
     /// must be used to decode the message, and then convert it with the reader schema.
-    pub writer_schema_cache: Option<Arc<ConfluentSchemaCache>>,
+    writer_schema_cache: WriterSchemaCache,
 
-    pub map_handling: Option<MapHandling>,
+    map_handling: Option<MapHandling>,
+}
+
+#[derive(Debug, Clone)]
+enum WriterSchemaCache {
+    Confluent(Arc<ConfluentSchemaCache>),
+    Glue(Arc<GlueSchemaCacheImpl>),
+    File,
 }
 
 impl AvroParserConfig {
     pub async fn new(encoding_properties: EncodingProperties) -> ConnectorResult<Self> {
         let AvroProperties {
-            use_schema_registry,
-            row_schema_location: schema_location,
-            client_config,
-            aws_auth_props,
-            topic,
-            enable_upsert,
+            schema_location,
             record_name,
             key_record_name,
-            name_strategy,
             map_handling,
         } = try_match_expand!(encoding_properties, EncodingProperties::Avro)?;
-        let url = handle_sr_list(schema_location.as_str())?;
-        if use_schema_registry {
-            let client = Client::new(url, &client_config)?;
-            let resolver = ConfluentSchemaCache::new(client);
+        match schema_location {
+            SchemaLocation::Confluent {
+                urls: schema_location,
+                client_config,
+                name_strategy,
+                topic,
+            } => {
+                let url = handle_sr_list(schema_location.as_str())?;
+                let client = Client::new(url, &client_config)?;
+                let resolver = ConfluentSchemaCache::new(client);
 
-            let subject_key = if enable_upsert {
-                Some(get_subject_by_strategy(
-                    &name_strategy,
-                    topic.as_str(),
-                    key_record_name.as_deref(),
-                    true,
-                )?)
-            } else {
                 if let Some(name) = &key_record_name {
                     bail!("unused FORMAT ENCODE option: key.message='{name}'");
                 }
-                None
-            };
-            let subject_value = get_subject_by_strategy(
-                &name_strategy,
-                topic.as_str(),
-                record_name.as_deref(),
-                false,
-            )?;
-            tracing::debug!("infer key subject {subject_key:?}, value subject {subject_value}");
-
-            Ok(Self {
-                schema: Arc::new(ResolvedAvroSchema::create(
-                    resolver.get_by_subject(&subject_value).await?,
-                )?),
-                key_schema: if let Some(subject_key) = subject_key {
-                    Some(Arc::new(ResolvedAvroSchema::create(
-                        resolver.get_by_subject(&subject_key).await?,
-                    )?))
-                } else {
-                    None
-                },
-                writer_schema_cache: Some(Arc::new(resolver)),
-                map_handling,
-            })
-        } else {
-            if enable_upsert {
-                bail!("avro upsert without schema registry is not supported");
+                let subject_value = get_subject_by_strategy(
+                    &name_strategy,
+                    topic.as_str(),
+                    record_name.as_deref(),
+                    false,
+                )?;
+                tracing::debug!("value subject {subject_value}");
+
+                Ok(Self {
+                    schema: Arc::new(ResolvedAvroSchema::create(
+                        resolver.get_by_subject(&subject_value).await?,
+                    )?),
+                    writer_schema_cache: WriterSchemaCache::Confluent(Arc::new(resolver)),
+                    map_handling,
+                })
+            }
+            SchemaLocation::File {
+                url: schema_location,
+                aws_auth_props,
+            } => {
+                let url = handle_sr_list(schema_location.as_str())?;
+                let url = url.first().unwrap();
+                let schema_content = bytes_from_url(url, aws_auth_props.as_ref()).await?;
+                let schema = Schema::parse_reader(&mut schema_content.as_slice())
+                    .context("failed to parse avro schema")?;
+                Ok(Self {
+                    schema: Arc::new(ResolvedAvroSchema::create(Arc::new(schema))?),
+                    writer_schema_cache: WriterSchemaCache::File,
+                    map_handling,
+                })
+            }
+            SchemaLocation::Glue {
+                schema_arn,
+                aws_auth_props,
+                mock_config,
+            } => {
+                let resolver =
+                    GlueSchemaCacheImpl::new(&aws_auth_props, mock_config.as_deref()).await?;
+                let schema = resolver.get_by_name(&schema_arn).await?;
+                Ok(Self {
+                    schema: Arc::new(ResolvedAvroSchema::create(schema)?),
+                    writer_schema_cache: WriterSchemaCache::Glue(Arc::new(resolver)),
+                    map_handling,
+                })
             }
-            let url = url.first().unwrap();
-            let schema_content = bytes_from_url(url, aws_auth_props.as_ref()).await?;
-            let schema = Schema::parse_reader(&mut schema_content.as_slice())
-                .context("failed to parse avro schema")?;
-            Ok(Self {
-                schema: Arc::new(ResolvedAvroSchema::create(Arc::new(schema))?),
-                key_schema: None,
-                writer_schema_cache: None,
-                map_handling,
-            })
         }
     }
 
@@ -185,46 +237,17 @@ impl AvroParserConfig {
 #[cfg(test)]
 mod test {
     use std::env;
-    use std::fs::OpenOptions;
-    use std::io::Write;
-    use std::ops::Sub;
-    use std::path::PathBuf;
 
-    use apache_avro::schema::RecordSchema;
-    use apache_avro::types::Record;
-    use apache_avro::{Codec, Days, Duration, Millis, Months, Writer};
-    use itertools::Itertools;
-    use risingwave_common::array::Op;
-    use risingwave_common::catalog::ColumnId;
-    use risingwave_common::row::Row;
-    use risingwave_common::types::{DataType, Date};
-    use risingwave_common::util::iter_util::ZipEqFast;
-    use risingwave_pb::catalog::StreamSourceInfo;
-    use risingwave_pb::plan_common::{PbEncodeType, PbFormatType};
     use url::Url;
 
     use super::*;
     use crate::connector_common::AwsAuthProps;
-    use crate::parser::plain_parser::PlainParser;
-    use crate::parser::{AccessBuilderImpl, SourceStreamChunkBuilder, SpecificParserConfig};
-    use crate::source::{SourceColumnDesc, SourceContext};
 
     fn test_data_path(file_name: &str) -> String {
         let curr_dir = env::current_dir().unwrap().into_os_string();
         curr_dir.into_string().unwrap() + "/src/test_data/" + file_name
     }
 
-    fn e2e_file_path(file_name: &str) -> String {
-        let curr_dir = env::current_dir().unwrap().into_os_string();
-        let binding = PathBuf::from(curr_dir);
-        let dir = binding.parent().unwrap().parent().unwrap();
-        dir.join("scripts/source/test_data/")
-            .join(file_name)
-            .to_str()
-            .unwrap()
-            .to_string()
-    }
-
     #[tokio::test]
     #[ignore]
     async fn test_load_schema_from_s3() {
@@ -261,290 +284,4 @@ mod test {
         assert!(schema.is_ok());
         println!("schema = {:?}", schema.unwrap());
     }
-
-    async fn new_avro_conf_from_local(file_name: &str) -> ConnectorResult<AvroParserConfig> {
-        let schema_path = format!("file://{}", test_data_path(file_name));
-        let info = StreamSourceInfo {
-            row_schema_location: schema_path.clone(),
-            use_schema_registry: false,
-            format: PbFormatType::Plain.into(),
-            row_encode: PbEncodeType::Avro.into(),
-            ..Default::default()
-        };
-        let parser_config = SpecificParserConfig::new(&info, &Default::default())?;
-        AvroParserConfig::new(parser_config.encoding_config).await
-    }
-
-    async fn new_avro_parser_from_local(file_name: &str) -> ConnectorResult<PlainParser> {
-        let conf = new_avro_conf_from_local(file_name).await?;
-
-        Ok(PlainParser {
-            key_builder: None,
-            payload_builder: AccessBuilderImpl::Avro(AvroAccessBuilder::new(
-                conf,
-                EncodingType::Value,
-            )?),
-            rw_columns: Vec::default(),
-            source_ctx: SourceContext::dummy().into(),
-            transaction_meta_builder: None,
-        })
-    }
-
-    #[tokio::test]
-    async fn test_avro_parser() {
-        let mut parser = new_avro_parser_from_local("simple-schema.avsc")
-            .await
-            .unwrap();
-        let builder = try_match_expand!(&parser.payload_builder, AccessBuilderImpl::Avro).unwrap();
-        let schema = builder.schema.original_schema.clone();
-        let record = build_avro_data(&schema);
-        assert_eq!(record.fields.len(), 11);
-        let mut writer = Writer::with_codec(&schema, Vec::new(), Codec::Snappy);
-        writer.append(record.clone()).unwrap();
-        let flush = writer.flush().unwrap();
-        assert!(flush > 0);
-        let input_data = writer.into_inner().unwrap();
-        let columns = build_rw_columns();
-        let mut builder = SourceStreamChunkBuilder::with_capacity(columns, 1);
-        {
-            let writer = builder.row_writer();
-            parser
-                .parse_inner(None, Some(input_data), writer)
-                .await
-                .unwrap();
-        }
-        let chunk = builder.finish();
-        let (op, row) = chunk.rows().next().unwrap();
-        assert_eq!(op, Op::Insert);
-
-        expect_test::expect![[r#"
-            ("id", Int(32)) => Some(Int32(32))
-            ("sequence_id", Long(64)) => Some(Int64(64))
-            ("name", Union(1, String("str_value"))) => Some(Utf8("str_value"))
-            ("score", Float(32.0)) => Some(Float32(OrderedFloat(32.0)))
-            ("avg_score", Double(64.0)) => Some(Float64(OrderedFloat(64.0)))
-            ("is_lasted", Boolean(true)) => Some(Bool(true))
-            ("entrance_date", Date(0)) => Some(Date(Date(1970-01-01)))
-            ("birthday", TimestampMillis(0)) => Some(Timestamptz(Timestamptz(0)))
-            ("anniversary", TimestampMicros(0)) => Some(Timestamptz(Timestamptz(0)))
-            ("passed", Duration(Duration { months: Months(1), days: Days(1), millis: Millis(1000) })) => Some(Interval(Interval { months: 1, days: 1, usecs: 1000000 }))
-            ("bytes", Bytes([1, 2, 3, 4, 5])) => Some(Bytea([1, 2, 3, 4, 5]))"#]].assert_eq(&format!(
-            "{}",
-            record
-                .fields
-                .iter()
-                .zip_eq_fast(row.iter())
-                .format_with("\n", |(avro, datum), f| {
-                    f(&format_args!("{:?} => {:?}", avro, datum))
-                })
-        ));
-    }
-
-    fn build_rw_columns() -> Vec<SourceColumnDesc> {
-        vec![
-            SourceColumnDesc::simple("id", DataType::Int32, ColumnId::from(0)),
-            SourceColumnDesc::simple("sequence_id", DataType::Int64, ColumnId::from(1)),
-            SourceColumnDesc::simple("name", DataType::Varchar, ColumnId::from(2)),
-            SourceColumnDesc::simple("score", DataType::Float32, ColumnId::from(3)),
-            SourceColumnDesc::simple("avg_score", DataType::Float64, ColumnId::from(4)),
-            SourceColumnDesc::simple("is_lasted", DataType::Boolean, ColumnId::from(5)),
-            SourceColumnDesc::simple("entrance_date", DataType::Date, ColumnId::from(6)),
-            SourceColumnDesc::simple("birthday", DataType::Timestamptz, ColumnId::from(7)),
-            SourceColumnDesc::simple("anniversary", DataType::Timestamptz, ColumnId::from(8)),
-            SourceColumnDesc::simple("passed", DataType::Interval, ColumnId::from(9)),
-            SourceColumnDesc::simple("bytes", DataType::Bytea, ColumnId::from(10)),
-        ]
-    }
-
-    fn build_field(schema: &Schema) -> Option<Value> {
-        match schema {
-            Schema::String => Some(Value::String("str_value".to_string())),
-            Schema::Int => Some(Value::Int(32_i32)),
-            Schema::Long => Some(Value::Long(64_i64)),
-            Schema::Float => Some(Value::Float(32_f32)),
-            Schema::Double => Some(Value::Double(64_f64)),
-            Schema::Boolean => Some(Value::Boolean(true)),
-            Schema::Bytes => Some(Value::Bytes(vec![1, 2, 3, 4, 5])),
-
-            Schema::Date => {
-                let original_date = Date::from_ymd_uncheck(1970, 1, 1).and_hms_uncheck(0, 0, 0);
-                let naive_date = Date::from_ymd_uncheck(1970, 1, 1).and_hms_uncheck(0, 0, 0);
-                let num_days = naive_date.0.sub(original_date.0).num_days() as i32;
-                Some(Value::Date(num_days))
-            }
-            Schema::TimestampMillis => {
-                let datetime = Date::from_ymd_uncheck(1970, 1, 1).and_hms_uncheck(0, 0, 0);
-                let timestamp_mills =
-                    Value::TimestampMillis(datetime.0.and_utc().timestamp() * 1_000);
-                Some(timestamp_mills)
-            }
-            Schema::TimestampMicros => {
-                let datetime = Date::from_ymd_uncheck(1970, 1, 1).and_hms_uncheck(0, 0, 0);
-                let timestamp_micros =
-                    Value::TimestampMicros(datetime.0.and_utc().timestamp() * 1_000_000);
-                Some(timestamp_micros)
-            }
-            Schema::Duration => {
-                let months = Months::new(1);
-                let days = Days::new(1);
-                let millis = Millis::new(1000);
-                Some(Value::Duration(Duration::new(months, days, millis)))
-            }
-
-            Schema::Union(union_schema) => {
-                let inner_schema = union_schema
-                    .variants()
-                    .iter()
-                    .find_or_first(|s| !matches!(s, &&Schema::Null))
-                    .unwrap();
-
-                match build_field(inner_schema) {
-                    None => {
-                        let index_of_union = union_schema
-                            .find_schema_with_known_schemata::<&Schema>(&Value::Null, None, &None)
-                            .unwrap()
-                            .0 as u32;
-                        Some(Value::Union(index_of_union, Box::new(Value::Null)))
-                    }
-                    Some(value) => {
-                        let index_of_union = union_schema
-                            .find_schema_with_known_schemata::<&Schema>(&value, None, &None)
-                            .unwrap()
-                            .0 as u32;
-                        Some(Value::Union(index_of_union, Box::new(value)))
-                    }
-                }
-            }
-            _ => None,
-        }
-    }
-
-    fn build_avro_data(schema: &Schema) -> Record<'_> {
-        let mut record = Record::new(schema).unwrap();
-        if let Schema::Record(RecordSchema {
-            name: _, fields, ..
-        }) = schema.clone()
-        {
-            for field in &fields {
-                let value = build_field(&field.schema)
-                    .unwrap_or_else(|| panic!("No value defined for field, {}", field.name));
-                record.put(field.name.as_str(), value)
-            }
-        }
-        record
-    }
-
-    #[tokio::test]
-    async fn test_map_to_columns() {
-        let conf = new_avro_conf_from_local("simple-schema.avsc")
-            .await
-            .unwrap();
-        let columns = conf.map_to_columns().unwrap();
-        assert_eq!(columns.len(), 11);
-        println!("{:?}", columns);
-    }
-
-    #[tokio::test]
-    async fn test_new_avro_parser() {
-        let avro_parser_rs = new_avro_parser_from_local("simple-schema.avsc").await;
-        let avro_parser = avro_parser_rs.unwrap();
-        println!("avro_parser = {:?}", avro_parser);
-    }
-
-    #[tokio::test]
-    async fn test_avro_union_type() {
-        let parser = new_avro_parser_from_local("union-schema.avsc")
-            .await
-            .unwrap();
-        let builder = try_match_expand!(&parser.payload_builder, AccessBuilderImpl::Avro).unwrap();
-        let schema = &builder.schema.original_schema;
-        let mut null_record = Record::new(schema).unwrap();
-        null_record.put("id", Value::Int(5));
-        null_record.put("age", Value::Union(0, Box::new(Value::Null)));
-        null_record.put("sequence_id", Value::Union(0, Box::new(Value::Null)));
-        null_record.put("name", Value::Union(0, Box::new(Value::Null)));
-        null_record.put("score", Value::Union(1, Box::new(Value::Null)));
-        null_record.put("avg_score", Value::Union(0, Box::new(Value::Null)));
-        null_record.put("is_lasted", Value::Union(0, Box::new(Value::Null)));
-        null_record.put("entrance_date", Value::Union(0, Box::new(Value::Null)));
-        null_record.put("birthday", Value::Union(0, Box::new(Value::Null)));
-        null_record.put("anniversary", Value::Union(0, Box::new(Value::Null)));
-
-        let mut writer = Writer::new(schema, Vec::new());
-        writer.append(null_record).unwrap();
-        writer.flush().unwrap();
-
-        let record = build_avro_data(schema);
-        writer.append(record).unwrap();
-        writer.flush().unwrap();
-
-        let records = writer.into_inner().unwrap();
-
-        let reader: Vec<_> = Reader::with_schema(schema, &records[..]).unwrap().collect();
-        assert_eq!(2, reader.len());
-        let null_record_expected: Vec<(String, Value)> = vec![
-            ("id".to_string(), Value::Int(5)),
-            ("age".to_string(), Value::Union(0, Box::new(Value::Null))),
-            (
-                "sequence_id".to_string(),
-                Value::Union(0, Box::new(Value::Null)),
-            ),
-            ("name".to_string(), Value::Union(0, Box::new(Value::Null))),
-            ("score".to_string(), Value::Union(1, Box::new(Value::Null))),
-            (
-                "avg_score".to_string(),
-                Value::Union(0, Box::new(Value::Null)),
-            ),
-            (
-                "is_lasted".to_string(),
-                Value::Union(0, Box::new(Value::Null)),
-            ),
-            (
-                "entrance_date".to_string(),
-                Value::Union(0, Box::new(Value::Null)),
-            ),
-            (
-                "birthday".to_string(),
-                Value::Union(0, Box::new(Value::Null)),
-            ),
-            (
-                "anniversary".to_string(),
-                Value::Union(0, Box::new(Value::Null)),
-            ),
-        ];
-        let null_record_value = reader.first().unwrap().as_ref().unwrap();
-        match null_record_value {
-            Value::Record(values) => {
-                assert_eq!(values, &null_record_expected)
-            }
-            _ => unreachable!(),
-        }
-    }
-
-    // run this script when updating `simple-schema.avsc`, the script will generate new value in
-    // `avro_simple_schema_bin.1`
-    #[ignore]
-    #[tokio::test]
-    async fn update_avro_payload() {
-        let conf = new_avro_conf_from_local("simple-schema.avsc")
-            .await
-            .unwrap();
-        let mut writer = Writer::new(conf.schema.original_schema.as_ref(), Vec::new());
-        let record = build_avro_data(conf.schema.original_schema.as_ref());
-        writer.append(record).unwrap();
-        let encoded = writer.into_inner().unwrap();
-        println!("path = {:?}", e2e_file_path("avro_simple_schema_bin.1"));
-        let mut file = OpenOptions::new()
-            .read(true)
-            .write(true)
-            .create(true)
-            .truncate(true)
-            .open(e2e_file_path("avro_simple_schema_bin.1"))
-            .unwrap();
-        file.write_all(encoded.as_slice()).unwrap();
-        println!(
-            "encoded = {:?}",
-            String::from_utf8_lossy(encoded.as_slice())
-        );
-    }
 }
diff --git a/src/connector/src/parser/debezium/avro_parser.rs b/src/connector/src/parser/debezium/avro_parser.rs
index 1a40b87c9d498..dbf685d6b620c 100644
--- a/src/connector/src/parser/debezium/avro_parser.rs
+++ b/src/connector/src/parser/debezium/avro_parser.rs
@@ -19,16 +19,15 @@ use apache_avro::types::Value;
 use apache_avro::{from_avro_datum, Schema};
 use risingwave_common::try_match_expand;
 use risingwave_connector_codec::decoder::avro::{
-    avro_extract_field_schema, avro_schema_skip_union, avro_schema_to_column_descs, AvroAccess,
-    AvroParseOptions, ResolvedAvroSchema,
+    avro_extract_field_schema, avro_schema_skip_nullable_union, avro_schema_to_column_descs,
+    AvroAccess, AvroParseOptions, ResolvedAvroSchema,
 };
-use risingwave_pb::catalog::PbSchemaRegistryNameStrategy;
 use risingwave_pb::plan_common::ColumnDesc;
 
 use crate::error::ConnectorResult;
 use crate::parser::avro::ConfluentSchemaCache;
 use crate::parser::unified::AccessImpl;
-use crate::parser::{AccessBuilder, EncodingProperties, EncodingType};
+use crate::parser::{AccessBuilder, EncodingProperties, EncodingType, SchemaLocation};
 use crate::schema::schema_registry::{
     extract_schema_id, get_subject_by_strategy, handle_sr_list, Client,
 };
@@ -95,14 +94,19 @@ pub struct DebeziumAvroParserConfig {
 impl DebeziumAvroParserConfig {
     pub async fn new(encoding_config: EncodingProperties) -> ConnectorResult<Self> {
         let avro_config = try_match_expand!(encoding_config, EncodingProperties::Avro)?;
-        let schema_location = &avro_config.row_schema_location;
-        let client_config = &avro_config.client_config;
-        let kafka_topic = &avro_config.topic;
+        let SchemaLocation::Confluent {
+            urls: schema_location,
+            client_config,
+            name_strategy,
+            topic: kafka_topic,
+        } = &avro_config.schema_location
+        else {
+            unreachable!()
+        };
         let url = handle_sr_list(schema_location)?;
         let client = Client::new(url, client_config)?;
         let resolver = ConfluentSchemaCache::new(client);
 
-        let name_strategy = &PbSchemaRegistryNameStrategy::Unspecified;
         let key_subject = get_subject_by_strategy(name_strategy, kafka_topic, None, true)?;
         let val_subject = get_subject_by_strategy(name_strategy, kafka_topic, None, false)?;
         let key_schema = resolver.get_by_subject(&key_subject).await?;
@@ -125,8 +129,40 @@ impl DebeziumAvroParserConfig {
     }
 
     pub fn map_to_columns(&self) -> ConnectorResult<Vec<ColumnDesc>> {
+        // Refer to debezium_avro_msg_schema.avsc for how the schema looks like:
+
+        // "fields": [
+        // {
+        //     "name": "before",
+        //     "type": [
+        //         "null",
+        //         {
+        //             "type": "record",
+        //             "name": "Value",
+        //             "fields": [...],
+        //         }
+        //     ],
+        //     "default": null
+        // },
+        // {
+        //     "name": "after",
+        //     "type": [
+        //         "null",
+        //         "Value"
+        //     ],
+        //     "default": null
+        // },
+        // ...]
+
+        // Other fields are:
+        // - source: describes the source metadata for the event
+        // - op
+        // - ts_ms
+        // - transaction
+        // See <https://debezium.io/documentation/reference/stable/connectors/mysql.html#mysql-events>
+
         avro_schema_to_column_descs(
-            avro_schema_skip_union(avro_extract_field_schema(
+            avro_schema_skip_nullable_union(avro_extract_field_schema(
                 // FIXME: use resolved schema here.
                 // Currently it works because "after" refers to a subtree in "before",
                 // but in theory, inside "before" there could also be a reference.
@@ -158,6 +194,7 @@ mod tests {
     use super::*;
     use crate::parser::{DebeziumParser, SourceStreamChunkBuilder, SpecificParserConfig};
     use crate::source::{SourceColumnDesc, SourceContext};
+    use crate::WithOptionsSecResolved;
 
     const DEBEZIUM_AVRO_DATA: &[u8] = b"\x00\x00\x00\x00\x06\x00\x02\xd2\x0f\x0a\x53\x61\x6c\x6c\x79\x0c\x54\x68\x6f\x6d\x61\x73\x2a\x73\x61\x6c\x6c\x79\x2e\x74\x68\x6f\x6d\x61\x73\x40\x61\x63\x6d\x65\x2e\x63\x6f\x6d\x16\x32\x2e\x31\x2e\x32\x2e\x46\x69\x6e\x61\x6c\x0a\x6d\x79\x73\x71\x6c\x12\x64\x62\x73\x65\x72\x76\x65\x72\x31\xc0\xb4\xe8\xb7\xc9\x61\x00\x30\x66\x69\x72\x73\x74\x5f\x69\x6e\x5f\x64\x61\x74\x61\x5f\x63\x6f\x6c\x6c\x65\x63\x74\x69\x6f\x6e\x12\x69\x6e\x76\x65\x6e\x74\x6f\x72\x79\x00\x02\x12\x63\x75\x73\x74\x6f\x6d\x65\x72\x73\x00\x00\x20\x6d\x79\x73\x71\x6c\x2d\x62\x69\x6e\x2e\x30\x30\x30\x30\x30\x33\x8c\x06\x00\x00\x00\x02\x72\x02\x92\xc3\xe8\xb7\xc9\x61\x00";
 
@@ -227,7 +264,7 @@ mod tests {
 
         let outer_schema = get_outer_schema();
         let expected_inner_schema = Schema::parse_str(inner_shema_str).unwrap();
-        let extracted_inner_schema = avro_schema_skip_union(
+        let extracted_inner_schema = avro_schema_skip_nullable_union(
             avro_extract_field_schema(&outer_schema, Some("before")).unwrap(),
         )
         .unwrap();
@@ -318,7 +355,7 @@ mod tests {
     fn test_map_to_columns() {
         let outer_schema = get_outer_schema();
         let columns = avro_schema_to_column_descs(
-            avro_schema_skip_union(
+            avro_schema_skip_nullable_union(
                 avro_extract_field_schema(&outer_schema, Some("before")).unwrap(),
             )
             .unwrap(),
@@ -363,7 +400,8 @@ mod tests {
             row_encode: PbEncodeType::Avro.into(),
             ..Default::default()
         };
-        let parser_config = SpecificParserConfig::new(&info, &props)?;
+        let parser_config =
+            SpecificParserConfig::new(&info, &WithOptionsSecResolved::without_secrets(props))?;
         let config = DebeziumAvroParserConfig::new(parser_config.clone().encoding_config).await?;
         let columns = config
             .map_to_columns()?
diff --git a/src/connector/src/parser/debezium/debezium_parser.rs b/src/connector/src/parser/debezium/debezium_parser.rs
index bc43e556cb4f6..3986593920770 100644
--- a/src/connector/src/parser/debezium/debezium_parser.rs
+++ b/src/connector/src/parser/debezium/debezium_parser.rs
@@ -75,9 +75,6 @@ async fn build_accessor_builder(
                     .unwrap_or(TimestamptzHandling::GuessNumberUnit),
             )?,
         )),
-        EncodingProperties::Protobuf(_) => {
-            Ok(AccessBuilderImpl::new_default(config, encoding_type).await?)
-        }
         _ => bail!("unsupported encoding for Debezium"),
     }
 }
diff --git a/src/connector/src/parser/debezium/mod.rs b/src/connector/src/parser/debezium/mod.rs
index 5b5416e647268..8852bfc25eb41 100644
--- a/src/connector/src/parser/debezium/mod.rs
+++ b/src/connector/src/parser/debezium/mod.rs
@@ -17,7 +17,9 @@
 mod avro_parser;
 mod debezium_parser;
 mod mongo_json_parser;
+pub mod schema_change;
 pub mod simd_json_parser;
+
 pub use avro_parser::*;
 pub use debezium_parser::*;
 pub use mongo_json_parser::DebeziumMongoJsonParser;
diff --git a/src/connector/src/parser/debezium/schema_change.rs b/src/connector/src/parser/debezium/schema_change.rs
new file mode 100644
index 0000000000000..4c61b52caaba9
--- /dev/null
+++ b/src/connector/src/parser/debezium/schema_change.rs
@@ -0,0 +1,94 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use risingwave_common::catalog::ColumnCatalog;
+use risingwave_pb::ddl_service::table_schema_change::TableChangeType as PbTableChangeType;
+use risingwave_pb::ddl_service::{
+    SchemaChangeEnvelope as PbSchemaChangeEnvelope, TableSchemaChange as PbTableSchemaChange,
+};
+
+#[derive(Debug)]
+pub struct SchemaChangeEnvelope {
+    pub table_changes: Vec<TableSchemaChange>,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub(crate) enum TableChangeType {
+    Unspecified,
+    Alter,
+    Create,
+    Drop,
+}
+
+impl TableChangeType {
+    #[allow(dead_code)]
+    pub fn from_proto(value: PbTableChangeType) -> Self {
+        match value {
+            PbTableChangeType::Alter => TableChangeType::Alter,
+            PbTableChangeType::Create => TableChangeType::Create,
+            PbTableChangeType::Drop => TableChangeType::Drop,
+            PbTableChangeType::Unspecified => TableChangeType::Unspecified,
+        }
+    }
+
+    pub fn to_proto(self) -> PbTableChangeType {
+        match self {
+            TableChangeType::Alter => PbTableChangeType::Alter,
+            TableChangeType::Create => PbTableChangeType::Create,
+            TableChangeType::Drop => PbTableChangeType::Drop,
+            TableChangeType::Unspecified => PbTableChangeType::Unspecified,
+        }
+    }
+}
+
+impl From<&str> for TableChangeType {
+    fn from(value: &str) -> Self {
+        match value {
+            "ALTER" => TableChangeType::Alter,
+            "CREATE" => TableChangeType::Create,
+            "DROP" => TableChangeType::Drop,
+            _ => TableChangeType::Unspecified,
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct TableSchemaChange {
+    pub(crate) cdc_table_name: String,
+    pub(crate) columns: Vec<ColumnCatalog>,
+    pub(crate) change_type: TableChangeType,
+}
+
+impl SchemaChangeEnvelope {
+    pub fn to_protobuf(&self) -> PbSchemaChangeEnvelope {
+        let table_changes = self
+            .table_changes
+            .iter()
+            .map(|table_change| {
+                let columns = table_change
+                    .columns
+                    .iter()
+                    .map(|column| column.to_protobuf())
+                    .collect();
+                PbTableSchemaChange {
+                    change_type: table_change.change_type.to_proto() as _,
+                    cdc_table_name: table_change.cdc_table_name.clone(),
+                    columns,
+                }
+            })
+            .collect();
+
+        PbSchemaChangeEnvelope { table_changes }
+    }
+}
diff --git a/src/connector/src/parser/debezium/simd_json_parser.rs b/src/connector/src/parser/debezium/simd_json_parser.rs
index f9738eb9e357e..08dd4ef7c2bdc 100644
--- a/src/connector/src/parser/debezium/simd_json_parser.rs
+++ b/src/connector/src/parser/debezium/simd_json_parser.rs
@@ -37,6 +37,13 @@ impl DebeziumJsonAccessBuilder {
             json_parse_options: JsonParseOptions::new_for_debezium(timestamptz_handling),
         })
     }
+
+    pub fn new_for_schema_event() -> ConnectorResult<Self> {
+        Ok(Self {
+            value: None,
+            json_parse_options: JsonParseOptions::default(),
+        })
+    }
 }
 
 impl AccessBuilder for DebeziumJsonAccessBuilder {
diff --git a/src/connector/src/parser/maxwell/maxwell_parser.rs b/src/connector/src/parser/maxwell/maxwell_parser.rs
index 8ba95ad212130..1f13c8a8bc30a 100644
--- a/src/connector/src/parser/maxwell/maxwell_parser.rs
+++ b/src/connector/src/parser/maxwell/maxwell_parser.rs
@@ -19,7 +19,7 @@ use crate::only_parse_payload;
 use crate::parser::unified::maxwell::MaxwellChangeEvent;
 use crate::parser::unified::util::apply_row_operation_on_stream_chunk_writer;
 use crate::parser::{
-    AccessBuilderImpl, ByteStreamSourceParser, EncodingProperties, EncodingType, ParserFormat,
+    AccessBuilderImpl, ByteStreamSourceParser, EncodingProperties, ParserFormat,
     SourceStreamChunkRowWriter, SpecificParserConfig,
 };
 use crate::source::{SourceColumnDesc, SourceContext, SourceContextRef};
@@ -39,9 +39,7 @@ impl MaxwellParser {
     ) -> ConnectorResult<Self> {
         match props.encoding_config {
             EncodingProperties::Json(_) => {
-                let payload_builder =
-                    AccessBuilderImpl::new_default(props.encoding_config, EncodingType::Value)
-                        .await?;
+                let payload_builder = AccessBuilderImpl::new_default(props.encoding_config).await?;
                 Ok(Self {
                     payload_builder,
                     rw_columns,
diff --git a/src/connector/src/parser/mod.rs b/src/connector/src/parser/mod.rs
index bca1b727d66a8..6cd843c0384b2 100644
--- a/src/connector/src/parser/mod.rs
+++ b/src/connector/src/parser/mod.rs
@@ -12,7 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::BTreeMap;
 use std::fmt::Debug;
 use std::sync::LazyLock;
 
@@ -24,12 +23,14 @@ pub use debezium::*;
 use futures::{Future, TryFutureExt};
 use futures_async_stream::try_stream;
 pub use json_parser::*;
+pub use parquet_parser::ParquetParser;
 pub use protobuf::*;
 use risingwave_common::array::{ArrayBuilderImpl, Op, StreamChunk};
 use risingwave_common::bail;
 use risingwave_common::catalog::{KAFKA_TIMESTAMP_COLUMN_NAME, TABLE_NAME_COLUMN_NAME};
 use risingwave_common::log::LogSuppresser;
 use risingwave_common::metrics::GLOBAL_ERROR_METRICS;
+use risingwave_common::secret::LocalSecretManager;
 use risingwave_common::types::{Datum, DatumCow, DatumRef, ScalarRefImpl};
 use risingwave_common::util::iter_util::ZipEqFast;
 use risingwave_common::util::tracing::InstrumentStream;
@@ -46,6 +47,7 @@ pub use self::mysql::mysql_row_to_owned_row;
 use self::plain_parser::PlainParser;
 pub use self::postgres::postgres_row_to_owned_row;
 use self::simd_json_parser::DebeziumJsonAccessBuilder;
+pub use self::sql_server::{sql_server_row_to_owned_row, ScalarImplTiberiusWrapper};
 pub use self::unified::json::{JsonAccess, TimestamptzHandling};
 pub use self::unified::Access;
 use self::unified::AccessImpl;
@@ -60,11 +62,13 @@ use crate::parser::util::{
     extreact_timestamp_from_meta,
 };
 use crate::schema::schema_registry::SchemaRegistryAuth;
+use crate::schema::AWS_GLUE_SCHEMA_ARN_KEY;
 use crate::source::monitor::GLOBAL_SOURCE_METRICS;
 use crate::source::{
     extract_source_struct, BoxSourceStream, ChunkSourceStream, SourceColumnDesc, SourceColumnType,
     SourceContext, SourceContextRef, SourceEncode, SourceFormat, SourceMessage, SourceMeta,
 };
+use crate::with_options::WithOptionsSecResolved;
 
 pub mod additional_columns;
 mod avro;
@@ -76,8 +80,10 @@ mod debezium;
 mod json_parser;
 mod maxwell;
 mod mysql;
+pub mod parquet_parser;
 pub mod plain_parser;
 mod postgres;
+mod sql_server;
 
 mod protobuf;
 pub mod scalar_adapter;
@@ -85,8 +91,9 @@ mod unified;
 mod upsert_parser;
 mod util;
 
+use debezium::schema_change::SchemaChangeEnvelope;
 pub use debezium::DEBEZIUM_IGNORE_KEY;
-use risingwave_common::buffer::BitmapBuilder;
+use risingwave_common::bitmap::BitmapBuilder;
 pub use unified::{AccessError, AccessResult};
 
 /// A builder for building a [`StreamChunk`] from [`SourceColumnDesc`].
@@ -575,6 +582,9 @@ pub enum ParseResult {
     Rows,
     /// A transaction control message is parsed.
     TransactionControl(TransactionControl),
+
+    /// A schema change message is parsed.
+    SchemaChange(SchemaChangeEnvelope),
 }
 
 #[derive(Clone, Copy, Debug, PartialEq)]
@@ -751,7 +761,7 @@ async fn into_chunk_stream_inner<P: ByteStreamSourceParser>(
                 // report to promethus
                 GLOBAL_SOURCE_METRICS
                     .direct_cdc_event_lag_latency
-                    .with_label_values(&[&msg_meta.full_table_name])
+                    .with_guarded_label_values(&[&msg_meta.full_table_name])
                     .observe(lag_ms as f64);
             }
 
@@ -825,6 +835,10 @@ async fn into_chunk_stream_inner<P: ByteStreamSourceParser>(
                         }
                     }
                 },
+
+                Ok(ParseResult::SchemaChange(_)) => {
+                    // TODO
+                }
             }
         }
 
@@ -842,6 +856,7 @@ async fn into_chunk_stream_inner<P: ByteStreamSourceParser>(
     }
 }
 
+/// Parses raw bytes into a specific format (avro, json, protobuf, ...), and then builds an [`Access`] from the parsed data.
 pub trait AccessBuilder {
     async fn generate_accessor(&mut self, payload: Vec<u8>) -> ConnectorResult<AccessImpl<'_>>;
 }
@@ -864,14 +879,11 @@ pub enum AccessBuilderImpl {
 }
 
 impl AccessBuilderImpl {
-    pub async fn new_default(
-        config: EncodingProperties,
-        kv: EncodingType,
-    ) -> ConnectorResult<Self> {
+    pub async fn new_default(config: EncodingProperties) -> ConnectorResult<Self> {
         let accessor = match config {
             EncodingProperties::Avro(_) => {
                 let config = AvroParserConfig::new(config).await?;
-                AccessBuilderImpl::Avro(AvroAccessBuilder::new(config, kv)?)
+                AccessBuilderImpl::Avro(AvroAccessBuilder::new(config)?)
             }
             EncodingProperties::Protobuf(_) => {
                 let config = ProtobufParserConfig::new(config).await?;
@@ -1066,18 +1078,47 @@ impl SpecificParserConfig {
 
 #[derive(Debug, Default, Clone)]
 pub struct AvroProperties {
-    pub use_schema_registry: bool,
-    pub row_schema_location: String,
-    pub client_config: SchemaRegistryAuth,
-    pub aws_auth_props: Option<AwsAuthProps>,
-    pub topic: String,
-    pub enable_upsert: bool,
+    pub schema_location: SchemaLocation,
     pub record_name: Option<String>,
     pub key_record_name: Option<String>,
-    pub name_strategy: PbSchemaRegistryNameStrategy,
     pub map_handling: Option<MapHandling>,
 }
 
+/// WIP: may cover protobuf and json schema later.
+#[derive(Debug, Clone)]
+pub enum SchemaLocation {
+    /// Avsc from `https://`, `s3://` or `file://`.
+    File {
+        url: String,
+        aws_auth_props: Option<AwsAuthProps>, // for s3
+    },
+    /// <https://docs.confluent.io/platform/current/schema-registry/index.html>
+    Confluent {
+        urls: String,
+        client_config: SchemaRegistryAuth,
+        name_strategy: PbSchemaRegistryNameStrategy,
+        topic: String,
+    },
+    /// <https://docs.aws.amazon.com/glue/latest/dg/schema-registry.html>
+    Glue {
+        schema_arn: String,
+        aws_auth_props: AwsAuthProps,
+        // When `Some(_)`, ignore AWS and load schemas from provided config
+        mock_config: Option<String>,
+    },
+}
+
+// TODO: `SpecificParserConfig` shall not `impl`/`derive` a `Default`
+impl Default for SchemaLocation {
+    fn default() -> Self {
+        // backward compatible but undesired
+        Self::File {
+            url: Default::default(),
+            aws_auth_props: None,
+        }
+    }
+}
+
 #[derive(Debug, Default, Clone)]
 pub struct ProtobufProperties {
     pub message_name: String,
@@ -1116,6 +1157,7 @@ pub enum EncodingProperties {
     Json(JsonProperties),
     MongoJson,
     Bytes(BytesProperties),
+    Parquet,
     Native,
     /// Encoding can't be specified because the source will determines it. Now only used in Iceberg.
     None,
@@ -1142,9 +1184,15 @@ impl SpecificParserConfig {
     // The validity of (format, encode) is ensured by `extract_format_encode`
     pub fn new(
         info: &StreamSourceInfo,
-        with_properties: &BTreeMap<String, String>,
+        with_properties: &WithOptionsSecResolved,
     ) -> ConnectorResult<Self> {
-        let source_struct = extract_source_struct(info)?;
+        let info = info.clone();
+        let source_struct = extract_source_struct(&info)?;
+        let format_encode_options_with_secret = LocalSecretManager::global()
+            .fill_secrets(info.format_encode_options, info.format_encode_secret_refs)?;
+        let (options, secret_refs) = with_properties.clone().into_parts();
+        let options_with_secret =
+            LocalSecretManager::global().fill_secrets(options.clone(), secret_refs.clone())?;
         let format = source_struct.format;
         let encode = source_struct.encode;
         // this transformation is needed since there may be config for the protocol
@@ -1153,7 +1201,7 @@ impl SpecificParserConfig {
             SourceFormat::Native => ProtocolProperties::Native,
             SourceFormat::None => ProtocolProperties::None,
             SourceFormat::Debezium => {
-                let debezium_props = DebeziumProps::from(&info.format_encode_options);
+                let debezium_props = DebeziumProps::from(&format_encode_options_with_secret);
                 ProtocolProperties::Debezium(debezium_props)
             }
             SourceFormat::DebeziumMongo => ProtocolProperties::DebeziumMongo,
@@ -1169,6 +1217,7 @@ impl SpecificParserConfig {
                 delimiter: info.csv_delimiter as u8,
                 has_header: info.csv_has_header,
             }),
+            (SourceFormat::Plain, SourceEncode::Parquet) => EncodingProperties::Parquet,
             (SourceFormat::Plain, SourceEncode::Avro)
             | (SourceFormat::Upsert, SourceEncode::Avro) => {
                 let mut config = AvroProperties {
@@ -1178,27 +1227,47 @@ impl SpecificParserConfig {
                         Some(info.proto_message_name.clone())
                     },
                     key_record_name: info.key_message_name.clone(),
-                    name_strategy: PbSchemaRegistryNameStrategy::try_from(info.name_strategy)
-                        .unwrap(),
-                    use_schema_registry: info.use_schema_registry,
-                    row_schema_location: info.row_schema_location.clone(),
-                    map_handling: MapHandling::from_options(&info.format_encode_options)?,
+                    map_handling: MapHandling::from_options(&format_encode_options_with_secret)?,
                     ..Default::default()
                 };
-                if format == SourceFormat::Upsert {
-                    config.enable_upsert = true;
-                }
-                if info.use_schema_registry {
-                    config.topic.clone_from(get_kafka_topic(with_properties)?);
-                    config.client_config = SchemaRegistryAuth::from(&info.format_encode_options);
-                } else {
-                    config.aws_auth_props = Some(
-                        serde_json::from_value::<AwsAuthProps>(
-                            serde_json::to_value(info.format_encode_options.clone()).unwrap(),
+                config.schema_location = if let Some(schema_arn) =
+                    format_encode_options_with_secret.get(AWS_GLUE_SCHEMA_ARN_KEY)
+                {
+                    risingwave_common::license::Feature::GlueSchemaRegistry
+                        .check_available()
+                        .map_err(anyhow::Error::from)?;
+                    SchemaLocation::Glue {
+                        schema_arn: schema_arn.clone(),
+                        aws_auth_props: serde_json::from_value::<AwsAuthProps>(
+                            serde_json::to_value(format_encode_options_with_secret.clone())
+                                .unwrap(),
                         )
                         .map_err(|e| anyhow::anyhow!(e))?,
-                    );
-                }
+                        // The option `mock_config` is not public and we can break compatibility.
+                        mock_config: format_encode_options_with_secret
+                            .get("aws.glue.mock_config")
+                            .cloned(),
+                    }
+                } else if info.use_schema_registry {
+                    SchemaLocation::Confluent {
+                        urls: info.row_schema_location.clone(),
+                        client_config: SchemaRegistryAuth::from(&format_encode_options_with_secret),
+                        name_strategy: PbSchemaRegistryNameStrategy::try_from(info.name_strategy)
+                            .unwrap(),
+                        topic: get_kafka_topic(with_properties)?.clone(),
+                    }
+                } else {
+                    SchemaLocation::File {
+                        url: info.row_schema_location.clone(),
+                        aws_auth_props: Some(
+                            serde_json::from_value::<AwsAuthProps>(
+                                serde_json::to_value(format_encode_options_with_secret.clone())
+                                    .unwrap(),
+                            )
+                            .map_err(|e| anyhow::anyhow!(e))?,
+                        ),
+                    }
+                };
                 EncodingProperties::Avro(config)
             }
             (SourceFormat::Plain, SourceEncode::Protobuf)
@@ -1219,12 +1288,16 @@ impl SpecificParserConfig {
                     config.enable_upsert = true;
                 }
                 if info.use_schema_registry {
-                    config.topic.clone_from(get_kafka_topic(with_properties)?);
-                    config.client_config = SchemaRegistryAuth::from(&info.format_encode_options);
+                    config
+                        .topic
+                        .clone_from(get_kafka_topic(&options_with_secret)?);
+                    config.client_config =
+                        SchemaRegistryAuth::from(&format_encode_options_with_secret);
                 } else {
                     config.aws_auth_props = Some(
                         serde_json::from_value::<AwsAuthProps>(
-                            serde_json::to_value(info.format_encode_options.clone()).unwrap(),
+                            serde_json::to_value(format_encode_options_with_secret.clone())
+                                .unwrap(),
                         )
                         .map_err(|e| anyhow::anyhow!(e))?,
                     );
@@ -1238,12 +1311,14 @@ impl SpecificParserConfig {
                     } else {
                         Some(info.proto_message_name.clone())
                     },
-                    name_strategy: PbSchemaRegistryNameStrategy::try_from(info.name_strategy)
-                        .unwrap(),
                     key_record_name: info.key_message_name.clone(),
-                    row_schema_location: info.row_schema_location.clone(),
-                    topic: get_kafka_topic(with_properties).unwrap().clone(),
-                    client_config: SchemaRegistryAuth::from(&info.format_encode_options),
+                    schema_location: SchemaLocation::Confluent {
+                        urls: info.row_schema_location.clone(),
+                        client_config: SchemaRegistryAuth::from(&format_encode_options_with_secret),
+                        name_strategy: PbSchemaRegistryNameStrategy::try_from(info.name_strategy)
+                            .unwrap(),
+                        topic: get_kafka_topic(with_properties).unwrap().clone(),
+                    },
                     ..Default::default()
                 })
             }
@@ -1257,7 +1332,7 @@ impl SpecificParserConfig {
             ) => EncodingProperties::Json(JsonProperties {
                 use_schema_registry: info.use_schema_registry,
                 timestamptz_handling: TimestamptzHandling::from_options(
-                    &info.format_encode_options,
+                    &format_encode_options_with_secret,
                 )?,
             }),
             (SourceFormat::DebeziumMongo, SourceEncode::Json) => {
diff --git a/src/connector/src/parser/mysql.rs b/src/connector/src/parser/mysql.rs
index d1df27263e808..a28dddc9aa65a 100644
--- a/src/connector/src/parser/mysql.rs
+++ b/src/connector/src/parser/mysql.rs
@@ -149,7 +149,7 @@ mod tests {
     use mysql_async::Row as MySqlRow;
     use risingwave_common::catalog::{Field, Schema};
     use risingwave_common::row::Row;
-    use risingwave_common::types::{DataType, ToText};
+    use risingwave_common::types::DataType;
     use tokio_stream::StreamExt;
 
     use crate::parser::mysql_row_to_owned_row;
@@ -187,7 +187,7 @@ mod tests {
                 let d = owned_row.datum_at(2);
                 if let Some(scalar) = d {
                     let v = scalar.into_timestamptz();
-                    println!("timestamp: {}", v.to_text());
+                    println!("timestamp: {:?}", v);
                 }
             }
         }
diff --git a/src/connector/src/parser/parquet_parser.rs b/src/connector/src/parser/parquet_parser.rs
new file mode 100644
index 0000000000000..eeefaaf2d82b5
--- /dev/null
+++ b/src/connector/src/parser/parquet_parser.rs
@@ -0,0 +1,190 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+use std::sync::Arc;
+
+use arrow_array_iceberg::RecordBatch;
+use futures_async_stream::try_stream;
+use risingwave_common::array::arrow::IcebergArrowConvert;
+use risingwave_common::array::{ArrayBuilderImpl, DataChunk, StreamChunk};
+use risingwave_common::types::{Datum, ScalarImpl};
+
+use crate::parser::ConnectorResult;
+use crate::source::SourceColumnDesc;
+/// `ParquetParser` is responsible for converting the incoming `record_batch_stream`
+/// into a `streamChunk`.
+#[derive(Debug)]
+pub struct ParquetParser {
+    rw_columns: Vec<SourceColumnDesc>,
+    file_name: String,
+    offset: usize,
+}
+
+impl ParquetParser {
+    pub fn new(
+        rw_columns: Vec<SourceColumnDesc>,
+        file_name: String,
+        offset: usize,
+    ) -> ConnectorResult<Self> {
+        Ok(Self {
+            rw_columns,
+            file_name,
+            offset,
+        })
+    }
+
+    #[try_stream(boxed, ok = StreamChunk, error = crate::error::ConnectorError)]
+    pub async fn into_stream(
+        mut self,
+        record_batch_stream: parquet::arrow::async_reader::ParquetRecordBatchStream<
+            tokio_util::compat::Compat<opendal::FuturesAsyncReader>,
+        >,
+    ) {
+        #[for_await]
+        for record_batch in record_batch_stream {
+            let record_batch: RecordBatch = record_batch?;
+            // Convert each record batch into a stream chunk according to user defined schema.
+            let chunk: StreamChunk = self.convert_record_batch_to_stream_chunk(record_batch)?;
+
+            yield chunk;
+        }
+    }
+
+    fn inc_offset(&mut self) {
+        self.offset += 1;
+    }
+
+    /// The function `convert_record_batch_to_stream_chunk` is designed to transform the given `RecordBatch` into a `StreamChunk`.
+    ///
+    /// For each column in the source column:
+    /// - If the column's schema matches a column in the `RecordBatch` (both the data type and column name are the same),
+    ///   the corresponding records are converted into a column of the `StreamChunk`.
+    /// - If the column's schema does not match, null values are inserted.
+    /// - Hidden columns are handled separately by filling in the appropriate fields to ensure the data chunk maintains the correct format.
+    /// - If a column in the Parquet file does not exist in the source schema, it is skipped.
+    ///
+    /// # Arguments
+    ///
+    /// * `record_batch` - The `RecordBatch` to be converted into a `StreamChunk`.
+    ///
+    /// # Returns
+    ///
+    /// A `StreamChunk` containing the converted data from the `RecordBatch`.
+
+    // The hidden columns that must be included here are _rw_file and _rw_offset.
+    // Depending on whether the user specifies a primary key (pk), there may be an additional hidden column row_id.
+    // Therefore, the maximum number of hidden columns is three.
+
+    fn convert_record_batch_to_stream_chunk(
+        &mut self,
+        record_batch: RecordBatch,
+    ) -> Result<StreamChunk, crate::error::ConnectorError> {
+        const MAX_HIDDEN_COLUMN_NUMS: usize = 3;
+        let column_size = self.rw_columns.len();
+        let mut chunk_columns = Vec::with_capacity(self.rw_columns.len() + MAX_HIDDEN_COLUMN_NUMS);
+        for source_column in self.rw_columns.clone() {
+            match source_column.column_type {
+                crate::source::SourceColumnType::Normal => {
+                    match source_column.is_hidden_addition_col {
+                        false => {
+                            let rw_data_type = &source_column.data_type;
+                            let rw_column_name = &source_column.name;
+                            if let Some(parquet_column) =
+                                record_batch.column_by_name(rw_column_name)
+                            {
+                                let arrow_field = IcebergArrowConvert
+                                    .to_arrow_field(rw_column_name, rw_data_type)?;
+                                let converted_arrow_data_type: &arrow_schema_iceberg::DataType =
+                                    arrow_field.data_type();
+                                if converted_arrow_data_type == parquet_column.data_type() {
+                                    let array_impl = IcebergArrowConvert
+                                        .array_from_arrow_array(&arrow_field, parquet_column)?;
+                                    let column = Arc::new(array_impl);
+                                    chunk_columns.push(column);
+                                } else {
+                                    // data type mismatch, this column is set to null.
+                                    let mut array_builder = ArrayBuilderImpl::with_type(
+                                        column_size,
+                                        rw_data_type.clone(),
+                                    );
+
+                                    array_builder.append_n_null(record_batch.num_rows());
+                                    let res = array_builder.finish();
+                                    let column = Arc::new(res);
+                                    chunk_columns.push(column);
+                                }
+                            } else {
+                                // For columns defined in the source schema but not present in the Parquet file, null values are filled in.
+                                let mut array_builder =
+                                    ArrayBuilderImpl::with_type(column_size, rw_data_type.clone());
+
+                                array_builder.append_n_null(record_batch.num_rows());
+                                let res = array_builder.finish();
+                                let column = Arc::new(res);
+                                chunk_columns.push(column);
+                            }
+                        }
+                        // handle hidden columns, for file source, the hidden columns are only `Offset` and `Filename`
+                        true => {
+                            if let Some(additional_column_type) =
+                                &source_column.additional_column.column_type
+                            {
+                                match additional_column_type{
+                                risingwave_pb::plan_common::additional_column::ColumnType::Offset(_) =>{
+                                    let mut array_builder =
+                                    ArrayBuilderImpl::with_type(column_size, source_column.data_type.clone());
+                                    for _ in 0..record_batch.num_rows(){
+                                        let datum: Datum = Some(ScalarImpl::Utf8((self.offset).to_string().into()));
+                                        self.inc_offset();
+                                        array_builder.append(datum);
+                                    }
+                                    let res = array_builder.finish();
+                                    let column = Arc::new(res);
+                                    chunk_columns.push(column);
+
+                                },
+                                risingwave_pb::plan_common::additional_column::ColumnType::Filename(_) => {
+                                    let mut array_builder =
+                                    ArrayBuilderImpl::with_type(column_size, source_column.data_type.clone());
+                                    let datum: Datum =  Some(ScalarImpl::Utf8(self.file_name.clone().into()));
+                                    array_builder.append_n(record_batch.num_rows(), datum);
+                                    let res = array_builder.finish();
+                                    let column = Arc::new(res);
+                                    chunk_columns.push(column);
+                                },
+                                _ => unreachable!()
+                            }
+                            }
+                        }
+                    }
+                }
+                crate::source::SourceColumnType::RowId => {
+                    let mut array_builder =
+                        ArrayBuilderImpl::with_type(column_size, source_column.data_type.clone());
+                    let datum: Datum = None;
+                    array_builder.append_n(record_batch.num_rows(), datum);
+                    let res = array_builder.finish();
+                    let column = Arc::new(res);
+                    chunk_columns.push(column);
+                }
+                // The following fields is only used in CDC source
+                crate::source::SourceColumnType::Offset | crate::source::SourceColumnType::Meta => {
+                    unreachable!()
+                }
+            }
+        }
+
+        let data_chunk = DataChunk::new(chunk_columns.clone(), record_batch.num_rows());
+        Ok(data_chunk.into())
+    }
+}
diff --git a/src/connector/src/parser/plain_parser.rs b/src/connector/src/parser/plain_parser.rs
index f34fd29837174..663fcb30e6ac9 100644
--- a/src/connector/src/parser/plain_parser.rs
+++ b/src/connector/src/parser/plain_parser.rs
@@ -17,16 +17,17 @@ use risingwave_common::bail;
 use super::unified::json::TimestamptzHandling;
 use super::unified::kv_event::KvEvent;
 use super::{
-    AccessBuilderImpl, ByteStreamSourceParser, EncodingProperties, EncodingType,
-    SourceStreamChunkRowWriter, SpecificParserConfig,
+    AccessBuilderImpl, ByteStreamSourceParser, EncodingProperties, SourceStreamChunkRowWriter,
+    SpecificParserConfig,
 };
 use crate::error::ConnectorResult;
 use crate::parser::bytes_parser::BytesAccessBuilder;
 use crate::parser::simd_json_parser::DebeziumJsonAccessBuilder;
-use crate::parser::unified::debezium::parse_transaction_meta;
+use crate::parser::unified::debezium::{parse_schema_change, parse_transaction_meta};
 use crate::parser::unified::AccessImpl;
 use crate::parser::upsert_parser::get_key_column_name;
 use crate::parser::{BytesProperties, ParseResult, ParserFormat};
+use crate::source::cdc::CdcMessageType;
 use crate::source::{SourceColumnDesc, SourceContext, SourceContextRef, SourceMeta};
 
 /// Parser for `FORMAT PLAIN`, i.e., append-only source.
@@ -38,6 +39,7 @@ pub struct PlainParser {
     pub source_ctx: SourceContextRef,
     // parsing transaction metadata for shared cdc source
     pub transaction_meta_builder: Option<AccessBuilderImpl>,
+    pub schema_change_builder: Option<AccessBuilderImpl>,
 }
 
 impl PlainParser {
@@ -61,7 +63,7 @@ impl PlainParser {
             | EncodingProperties::Protobuf(_)
             | EncodingProperties::Avro(_)
             | EncodingProperties::Bytes(_) => {
-                AccessBuilderImpl::new_default(props.encoding_config, EncodingType::Value).await?
+                AccessBuilderImpl::new_default(props.encoding_config).await?
             }
             _ => bail!("Unsupported encoding for Plain"),
         };
@@ -69,12 +71,18 @@ impl PlainParser {
         let transaction_meta_builder = Some(AccessBuilderImpl::DebeziumJson(
             DebeziumJsonAccessBuilder::new(TimestamptzHandling::GuessNumberUnit)?,
         ));
+
+        let schema_change_builder = Some(AccessBuilderImpl::DebeziumJson(
+            DebeziumJsonAccessBuilder::new_for_schema_event()?,
+        ));
+
         Ok(Self {
             key_builder,
             payload_builder,
             rw_columns,
             source_ctx,
             transaction_meta_builder,
+            schema_change_builder,
         })
     }
 
@@ -82,26 +90,62 @@ impl PlainParser {
         &mut self,
         key: Option<Vec<u8>>,
         payload: Option<Vec<u8>>,
-        mut writer: SourceStreamChunkRowWriter<'_>,
+        writer: SourceStreamChunkRowWriter<'_>,
     ) -> ConnectorResult<ParseResult> {
-        // if the message is transaction metadata, parse it and return
+        // plain parser also used in the shared cdc source,
+        // we need to handle transaction metadata and schema change messages here
         if let Some(msg_meta) = writer.row_meta
             && let SourceMeta::DebeziumCdc(cdc_meta) = msg_meta.meta
-            && cdc_meta.is_transaction_meta
             && let Some(data) = payload
         {
-            let accessor = self
-                .transaction_meta_builder
-                .as_mut()
-                .expect("expect transaction metadata access builder")
-                .generate_accessor(data)
-                .await?;
-            return match parse_transaction_meta(&accessor, &self.source_ctx.connector_props) {
-                Ok(transaction_control) => Ok(ParseResult::TransactionControl(transaction_control)),
-                Err(err) => Err(err)?,
-            };
+            match cdc_meta.msg_type {
+                CdcMessageType::Data | CdcMessageType::Heartbeat => {
+                    return self.parse_rows(key, Some(data), writer).await;
+                }
+                CdcMessageType::TransactionMeta => {
+                    let accessor = self
+                        .transaction_meta_builder
+                        .as_mut()
+                        .expect("expect transaction metadata access builder")
+                        .generate_accessor(data)
+                        .await?;
+                    return match parse_transaction_meta(&accessor, &self.source_ctx.connector_props)
+                    {
+                        Ok(transaction_control) => {
+                            Ok(ParseResult::TransactionControl(transaction_control))
+                        }
+                        Err(err) => Err(err)?,
+                    };
+                }
+                CdcMessageType::SchemaChange => {
+                    let accessor = self
+                        .schema_change_builder
+                        .as_mut()
+                        .expect("expect schema change access builder")
+                        .generate_accessor(data)
+                        .await?;
+
+                    return match parse_schema_change(&accessor, &self.source_ctx.connector_props) {
+                        Ok(schema_change) => Ok(ParseResult::SchemaChange(schema_change)),
+                        Err(err) => Err(err)?,
+                    };
+                }
+                CdcMessageType::Unspecified => {
+                    unreachable!()
+                }
+            }
         }
 
+        // for non-cdc source messages
+        self.parse_rows(key, payload, writer).await
+    }
+
+    async fn parse_rows(
+        &mut self,
+        key: Option<Vec<u8>>,
+        payload: Option<Vec<u8>>,
+        mut writer: SourceStreamChunkRowWriter<'_>,
+    ) -> ConnectorResult<ParseResult> {
         let mut row_op: KvEvent<AccessImpl<'_>, AccessImpl<'_>> = KvEvent::default();
 
         if let Some(data) = key
@@ -158,11 +202,13 @@ mod tests {
     use std::ops::Deref;
     use std::sync::Arc;
 
+    use expect_test::expect;
     use futures::executor::block_on;
     use futures::StreamExt;
     use futures_async_stream::try_stream;
     use itertools::Itertools;
     use risingwave_common::catalog::{ColumnCatalog, ColumnDesc, ColumnId};
+    use risingwave_pb::connector_service::cdc_message;
 
     use super::*;
     use crate::parser::{MessageMeta, SourceStreamChunkBuilder, TransactionControl};
@@ -281,7 +327,11 @@ mod tests {
                     meta: SourceMeta::DebeziumCdc(DebeziumCdcMeta::new(
                         "orders".to_string(),
                         0,
-                        transactional,
+                        if transactional {
+                            cdc_message::CdcMessageType::TransactionMeta
+                        } else {
+                            cdc_message::CdcMessageType::Data
+                        },
                     )),
                     split_id: SplitId::from("1001"),
                     offset: "0".into(),
@@ -295,7 +345,7 @@ mod tests {
                     meta: SourceMeta::DebeziumCdc(DebeziumCdcMeta::new(
                         "orders".to_string(),
                         0,
-                        false,
+                        cdc_message::CdcMessageType::Data,
                     )),
                     split_id: SplitId::from("1001"),
                     offset: "0".into(),
@@ -309,7 +359,11 @@ mod tests {
                     meta: SourceMeta::DebeziumCdc(DebeziumCdcMeta::new(
                         "orders".to_string(),
                         0,
-                        transactional,
+                        if transactional {
+                            cdc_message::CdcMessageType::TransactionMeta
+                        } else {
+                            cdc_message::CdcMessageType::Data
+                        },
                     )),
                     split_id: SplitId::from("1001"),
                     offset: "0".into(),
@@ -355,7 +409,11 @@ mod tests {
         let begin_msg = r#"{"schema":null,"payload":{"status":"BEGIN","id":"3E11FA47-71CA-11E1-9E33-C80AA9429562:23","event_count":null,"data_collections":null,"ts_ms":1704269323180}}"#;
         let commit_msg = r#"{"schema":null,"payload":{"status":"END","id":"3E11FA47-71CA-11E1-9E33-C80AA9429562:23","event_count":11,"data_collections":[{"data_collection":"public.orders_tx","event_count":5},{"data_collection":"public.person","event_count":6}],"ts_ms":1704269323180}}"#;
 
-        let cdc_meta = SourceMeta::DebeziumCdc(DebeziumCdcMeta::new("orders".to_string(), 0, true));
+        let cdc_meta = SourceMeta::DebeziumCdc(DebeziumCdcMeta::new(
+            "orders".to_string(),
+            0,
+            cdc_message::CdcMessageType::TransactionMeta,
+        ));
         let msg_meta = MessageMeta {
             meta: &cdc_meta,
             split_id: "1001",
@@ -393,4 +451,120 @@ mod tests {
         let output = builder.take(10);
         assert_eq!(0, output.cardinality());
     }
+
+    #[tokio::test]
+    async fn test_parse_schema_change() {
+        let schema = vec![
+            ColumnCatalog {
+                column_desc: ColumnDesc::named("payload", ColumnId::placeholder(), DataType::Jsonb),
+                is_hidden: false,
+            },
+            ColumnCatalog::offset_column(),
+            ColumnCatalog::cdc_table_name_column(),
+        ];
+
+        let columns = schema
+            .iter()
+            .map(|c| SourceColumnDesc::from(&c.column_desc))
+            .collect::<Vec<_>>();
+
+        // format plain encode json parser
+        let source_ctx = SourceContext {
+            connector_props: ConnectorProperties::MysqlCdc(Box::default()),
+            ..SourceContext::dummy()
+        };
+        let mut parser = PlainParser::new(
+            SpecificParserConfig::DEFAULT_PLAIN_JSON,
+            columns.clone(),
+            Arc::new(source_ctx),
+        )
+        .await
+        .unwrap();
+        let mut builder = SourceStreamChunkBuilder::with_capacity(columns, 0);
+
+        let msg = r#"{"schema":null,"payload": { "databaseName": "mydb", "ddl": "ALTER TABLE test add column v2 varchar(32)", "schemaName": null, "source": { "connector": "mysql", "db": "mydb", "file": "binlog.000065", "gtid": null, "name": "RW_CDC_1", "pos": 234, "query": null, "row": 0, "sequence": null, "server_id": 1, "snapshot": "false", "table": "test", "thread": null, "ts_ms": 1718354727000, "version": "2.4.2.Final" }, "tableChanges": [ { "id": "\"mydb\".\"test\"", "table": { "columns": [ { "autoIncremented": false, "charsetName": null, "comment": null, "defaultValueExpression": null, "enumValues": null, "generated": false, "jdbcType": 4, "length": null, "name": "id", "nativeType": null, "optional": false, "position": 1, "scale": null, "typeExpression": "INT", "typeName": "INT" }, { "autoIncremented": false, "charsetName": null, "comment": null, "defaultValueExpression": null, "enumValues": null, "generated": false, "jdbcType": 2014, "length": null, "name": "v1", "nativeType": null, "optional": true, "position": 2, "scale": null, "typeExpression": "TIMESTAMP", "typeName": "TIMESTAMP" }, { "autoIncremented": false, "charsetName": "utf8mb4", "comment": null, "defaultValueExpression": null, "enumValues": null, "generated": false, "jdbcType": 12, "length": 32, "name": "v2", "nativeType": null, "optional": true, "position": 3, "scale": null, "typeExpression": "VARCHAR", "typeName": "VARCHAR" } ], "comment": null, "defaultCharsetName": "utf8mb4", "primaryKeyColumnNames": [ "id" ] }, "type": "ALTER" } ], "ts_ms": 1718354727594 }}"#;
+        let cdc_meta = SourceMeta::DebeziumCdc(DebeziumCdcMeta::new(
+            "mydb.test".to_string(),
+            0,
+            cdc_message::CdcMessageType::SchemaChange,
+        ));
+        let msg_meta = MessageMeta {
+            meta: &cdc_meta,
+            split_id: "1001",
+            offset: "",
+        };
+
+        let res = parser
+            .parse_one_with_txn(
+                None,
+                Some(msg.as_bytes().to_vec()),
+                builder.row_writer().with_meta(msg_meta),
+            )
+            .await;
+
+        let res = res.unwrap();
+        expect![[r#"
+            SchemaChange(
+                SchemaChangeEnvelope {
+                    table_changes: [
+                        TableSchemaChange {
+                            cdc_table_name: "mydb.test",
+                            columns: [
+                                ColumnCatalog {
+                                    column_desc: ColumnDesc {
+                                        data_type: Int32,
+                                        column_id: #2147483646,
+                                        name: "id",
+                                        field_descs: [],
+                                        type_name: "",
+                                        generated_or_default_column: None,
+                                        description: None,
+                                        additional_column: AdditionalColumn {
+                                            column_type: None,
+                                        },
+                                        version: Pr13707,
+                                    },
+                                    is_hidden: false,
+                                },
+                                ColumnCatalog {
+                                    column_desc: ColumnDesc {
+                                        data_type: Timestamptz,
+                                        column_id: #2147483646,
+                                        name: "v1",
+                                        field_descs: [],
+                                        type_name: "",
+                                        generated_or_default_column: None,
+                                        description: None,
+                                        additional_column: AdditionalColumn {
+                                            column_type: None,
+                                        },
+                                        version: Pr13707,
+                                    },
+                                    is_hidden: false,
+                                },
+                                ColumnCatalog {
+                                    column_desc: ColumnDesc {
+                                        data_type: Varchar,
+                                        column_id: #2147483646,
+                                        name: "v2",
+                                        field_descs: [],
+                                        type_name: "",
+                                        generated_or_default_column: None,
+                                        description: None,
+                                        additional_column: AdditionalColumn {
+                                            column_type: None,
+                                        },
+                                        version: Pr13707,
+                                    },
+                                    is_hidden: false,
+                                },
+                            ],
+                            change_type: Alter,
+                        },
+                    ],
+                },
+            )
+        "#]]
+        .assert_debug_eq(&res);
+    }
 }
diff --git a/src/connector/src/parser/sql_server.rs b/src/connector/src/parser/sql_server.rs
new file mode 100644
index 0000000000000..91f78b41ffb6b
--- /dev/null
+++ b/src/connector/src/parser/sql_server.rs
@@ -0,0 +1,297 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::sync::LazyLock;
+
+use chrono::{DateTime, NaiveDate, NaiveDateTime, NaiveTime, Utc};
+use risingwave_common::catalog::Schema;
+use risingwave_common::log::LogSuppresser;
+use risingwave_common::row::OwnedRow;
+use risingwave_common::types::{Date, Decimal, ScalarImpl, Time, Timestamp, Timestamptz};
+use rust_decimal::Decimal as RustDecimal;
+use thiserror_ext::AsReport;
+use tiberius::Row;
+
+use crate::parser::util::log_error;
+
+static LOG_SUPPERSSER: LazyLock<LogSuppresser> = LazyLock::new(LogSuppresser::default);
+
+pub fn sql_server_row_to_owned_row(row: &mut Row, schema: &Schema) -> OwnedRow {
+    let mut datums: Vec<Option<ScalarImpl>> = vec![];
+    for i in 0..schema.fields.len() {
+        let rw_field = &schema.fields[i];
+        let name = rw_field.name.as_str();
+        let datum = match row.try_get::<ScalarImplTiberiusWrapper, usize>(i) {
+            Ok(datum) => datum.map(|d| d.0),
+            Err(err) => {
+                log_error!(name, err, "parse column failed");
+                None
+            }
+        };
+        datums.push(datum);
+    }
+    OwnedRow::new(datums)
+}
+macro_rules! impl_tiberius_wrapper {
+    ($wrapper_name:ident, $variant_name:ident) => {
+        pub struct $wrapper_name($variant_name);
+
+        impl From<$variant_name> for $wrapper_name {
+            fn from(value: $variant_name) -> Self {
+                Self(value)
+            }
+        }
+    };
+}
+
+impl_tiberius_wrapper!(ScalarImplTiberiusWrapper, ScalarImpl);
+impl_tiberius_wrapper!(TimeTiberiusWrapper, Time);
+impl_tiberius_wrapper!(DateTiberiusWrapper, Date);
+impl_tiberius_wrapper!(TimestampTiberiusWrapper, Timestamp);
+impl_tiberius_wrapper!(TimestamptzTiberiusWrapper, Timestamptz);
+impl_tiberius_wrapper!(DecimalTiberiusWrapper, Decimal);
+
+macro_rules! impl_chrono_tiberius_wrapper {
+    ($wrapper_name:ident, $variant_name:ident, $chrono:ty) => {
+        impl<'a> tiberius::IntoSql<'a> for $wrapper_name {
+            fn into_sql(self) -> tiberius::ColumnData<'a> {
+                self.0 .0.into_sql()
+            }
+        }
+
+        impl<'a> tiberius::FromSql<'a> for $wrapper_name {
+            fn from_sql(
+                value: &'a tiberius::ColumnData<'static>,
+            ) -> tiberius::Result<Option<Self>> {
+                let instant = <$chrono as tiberius::FromSql>::from_sql(value)?;
+                let time = instant.map($variant_name::from).map($wrapper_name::from);
+                tiberius::Result::Ok(time)
+            }
+        }
+    };
+}
+
+impl_chrono_tiberius_wrapper!(TimeTiberiusWrapper, Time, NaiveTime);
+impl_chrono_tiberius_wrapper!(DateTiberiusWrapper, Date, NaiveDate);
+impl_chrono_tiberius_wrapper!(TimestampTiberiusWrapper, Timestamp, NaiveDateTime);
+
+impl<'a> tiberius::IntoSql<'a> for DecimalTiberiusWrapper {
+    fn into_sql(self) -> tiberius::ColumnData<'a> {
+        match self.0 {
+            Decimal::Normalized(d) => d.into_sql(),
+            Decimal::NaN => tiberius::ColumnData::Numeric(None),
+            Decimal::PositiveInf => tiberius::ColumnData::Numeric(None),
+            Decimal::NegativeInf => tiberius::ColumnData::Numeric(None),
+        }
+    }
+}
+
+impl<'a> tiberius::FromSql<'a> for DecimalTiberiusWrapper {
+    // TODO(kexiang): will sql server have inf/-inf/nan for decimal?
+    fn from_sql(value: &'a tiberius::ColumnData<'static>) -> tiberius::Result<Option<Self>> {
+        tiberius::Result::Ok(
+            <RustDecimal as tiberius::FromSql>::from_sql(value)?
+                .map(Decimal::Normalized)
+                .map(DecimalTiberiusWrapper::from),
+        )
+    }
+}
+
+impl<'a> tiberius::IntoSql<'a> for TimestamptzTiberiusWrapper {
+    fn into_sql(self) -> tiberius::ColumnData<'a> {
+        self.0.to_datetime_utc().into_sql()
+    }
+}
+
+impl<'a> tiberius::FromSql<'a> for TimestamptzTiberiusWrapper {
+    fn from_sql(value: &'a tiberius::ColumnData<'static>) -> tiberius::Result<Option<Self>> {
+        let instant = <DateTime<Utc> as tiberius::FromSql>::from_sql(value)?;
+        let time = instant
+            .map(Timestamptz::from)
+            .map(TimestamptzTiberiusWrapper::from);
+        tiberius::Result::Ok(time)
+    }
+}
+
+/// The following table shows the mapping between Rust types and Sql Server types in tiberius.
+/// |Rust Type|Sql Server Type|
+/// |`u8`|`tinyint`|
+/// |`i16`|`smallint`|
+/// |`i32`|`int`|
+/// |`i64`|`bigint`|
+/// |`f32`|`float(24)`|
+/// |`f64`|`float(53)`|
+/// |`bool`|`bit`|
+/// |`String`/`&str`|`nvarchar`/`varchar`/`nchar`/`char`/`ntext`/`text`|
+/// |`Vec<u8>`/`&[u8]`|`binary`/`varbinary`/`image`|
+/// |[`Uuid`]|`uniqueidentifier`|
+/// |[`Numeric`]|`numeric`/`decimal`|
+/// |[`Decimal`] (with feature flag `rust_decimal`)|`numeric`/`decimal`|
+/// |[`XmlData`]|`xml`|
+/// |[`NaiveDateTime`] (with feature flag `chrono`)|`datetime`/`datetime2`/`smalldatetime`|
+/// |[`NaiveDate`] (with feature flag `chrono`)|`date`|
+/// |[`NaiveTime`] (with feature flag `chrono`)|`time`|
+/// |[`DateTime`] (with feature flag `chrono`)|`datetimeoffset`|
+///
+/// See the [`time`] module for more information about the date and time structs.
+///
+/// [`Row#get`]: struct.Row.html#method.get
+/// [`Row#try_get`]: struct.Row.html#method.try_get
+/// [`time`]: time/index.html
+/// [`Uuid`]: struct.Uuid.html
+/// [`Numeric`]: numeric/struct.Numeric.html
+/// [`Decimal`]: numeric/struct.Decimal.html
+/// [`XmlData`]: xml/struct.XmlData.html
+/// [`NaiveDateTime`]: time/chrono/struct.NaiveDateTime.html
+/// [`NaiveDate`]: time/chrono/struct.NaiveDate.html
+/// [`NaiveTime`]: time/chrono/struct.NaiveTime.html
+/// [`DateTime`]: time/chrono/struct.DateTime.html
+impl<'a> tiberius::FromSql<'a> for ScalarImplTiberiusWrapper {
+    fn from_sql(value: &'a tiberius::ColumnData<'static>) -> tiberius::Result<Option<Self>> {
+        Ok(match &value {
+            tiberius::ColumnData::U8(_) => <u8 as tiberius::FromSql>::from_sql(value)?
+                .map(|v| ScalarImplTiberiusWrapper::from(ScalarImpl::from(v as i16))),
+            tiberius::ColumnData::I16(_) => <i16 as tiberius::FromSql>::from_sql(value)?
+                .map(ScalarImpl::from)
+                .map(ScalarImplTiberiusWrapper::from),
+            tiberius::ColumnData::I32(_) => <i32 as tiberius::FromSql>::from_sql(value)?
+                .map(ScalarImpl::from)
+                .map(ScalarImplTiberiusWrapper::from),
+            tiberius::ColumnData::I64(_) => <i64 as tiberius::FromSql>::from_sql(value)?
+                .map(ScalarImpl::from)
+                .map(ScalarImplTiberiusWrapper::from),
+            tiberius::ColumnData::F32(_) => <f32 as tiberius::FromSql>::from_sql(value)?
+                .map(ScalarImpl::from)
+                .map(ScalarImplTiberiusWrapper::from),
+            tiberius::ColumnData::F64(_) => <f64 as tiberius::FromSql>::from_sql(value)?
+                .map(ScalarImpl::from)
+                .map(ScalarImplTiberiusWrapper::from),
+            tiberius::ColumnData::Bit(_) => <bool as tiberius::FromSql>::from_sql(value)?
+                .map(ScalarImpl::from)
+                .map(ScalarImplTiberiusWrapper::from),
+            tiberius::ColumnData::String(_) => <&str as tiberius::FromSql>::from_sql(value)?
+                .map(ScalarImpl::from)
+                .map(ScalarImplTiberiusWrapper::from),
+            tiberius::ColumnData::Numeric(_) => {
+                <DecimalTiberiusWrapper as tiberius::FromSql>::from_sql(value)?
+                    .map(|w| ScalarImpl::from(w.0))
+                    .map(ScalarImplTiberiusWrapper::from)
+            }
+            tiberius::ColumnData::DateTime(_)
+            | tiberius::ColumnData::DateTime2(_)
+            | tiberius::ColumnData::SmallDateTime(_) => {
+                <TimestampTiberiusWrapper as tiberius::FromSql>::from_sql(value)?
+                    .map(|w| ScalarImpl::from(w.0))
+                    .map(ScalarImplTiberiusWrapper::from)
+            }
+            tiberius::ColumnData::Time(_) => {
+                <TimeTiberiusWrapper as tiberius::FromSql>::from_sql(value)?
+                    .map(|w| ScalarImpl::from(w.0))
+                    .map(ScalarImplTiberiusWrapper::from)
+            }
+            tiberius::ColumnData::Date(_) => {
+                <DateTiberiusWrapper as tiberius::FromSql>::from_sql(value)?
+                    .map(|w| ScalarImpl::from(w.0))
+                    .map(ScalarImplTiberiusWrapper::from)
+            }
+            tiberius::ColumnData::DateTimeOffset(_) => {
+                <TimestamptzTiberiusWrapper as tiberius::FromSql>::from_sql(value)?
+                    .map(|w| ScalarImpl::from(w.0))
+                    .map(ScalarImplTiberiusWrapper::from)
+            }
+            tiberius::ColumnData::Binary(_) => <&[u8] as tiberius::FromSql>::from_sql(value)?
+                .map(ScalarImpl::from)
+                .map(ScalarImplTiberiusWrapper::from),
+            tiberius::ColumnData::Guid(_) | tiberius::ColumnData::Xml(_) => {
+                return Err(tiberius::error::Error::Conversion(
+                    format!(
+                        "the sql server decoding for {:?} is unsupported",
+                        value.type_name()
+                    )
+                    .into(),
+                ))
+            }
+        })
+    }
+}
+
+/// The following table shows the mapping between Rust types and Sql Server types in tiberius.
+/// |Rust type|Sql Server type|
+/// |--------|--------|
+/// |`u8`|`tinyint`|
+/// |`i16`|`smallint`|
+/// |`i32`|`int`|
+/// |`i64`|`bigint`|
+/// |`f32`|`float(24)`|
+/// |`f64`|`float(53)`|
+/// |`bool`|`bit`|
+/// |`String`/`&str` (< 4000 characters)|`nvarchar(4000)`|
+/// |`String`/`&str`|`nvarchar(max)`|
+/// |`Vec<u8>`/`&[u8]` (< 8000 bytes)|`varbinary(8000)`|
+/// |`Vec<u8>`/`&[u8]`|`varbinary(max)`|
+/// |[`Uuid`]|`uniqueidentifier`|
+/// |[`Numeric`]|`numeric`/`decimal`|
+/// |[`Decimal`] (with feature flag `rust_decimal`)|`numeric`/`decimal`|
+/// |[`BigDecimal`] (with feature flag `bigdecimal`)|`numeric`/`decimal`|
+/// |[`XmlData`]|`xml`|
+/// |[`NaiveDate`] (with `chrono` feature, TDS 7.3 >)|`date`|
+/// |[`NaiveTime`] (with `chrono` feature, TDS 7.3 >)|`time`|
+/// |[`DateTime`] (with `chrono` feature, TDS 7.3 >)|`datetimeoffset`|
+/// |[`NaiveDateTime`] (with `chrono` feature, TDS 7.3 >)|`datetime2`|
+/// |[`NaiveDateTime`] (with `chrono` feature, TDS 7.2)|`datetime`|
+///
+/// It is possible to use some of the types to write into columns that are not
+/// of the same type. For example on systems following the TDS 7.3 standard (SQL
+/// Server 2008 and later), the chrono type `NaiveDateTime` can also be used to
+/// write to `datetime`, `datetime2` and `smalldatetime` columns. All string
+/// types can also be used with `ntext`, `text`, `varchar`, `nchar` and `char`
+/// columns. All binary types can also be used with `binary` and `image`
+/// columns.
+///
+/// See the [`time`] module for more information about the date and time structs.
+///
+/// [`Client#query`]: struct.Client.html#method.query
+/// [`Client#execute`]: struct.Client.html#method.execute
+/// [`time`]: time/index.html
+/// [`Uuid`]: struct.Uuid.html
+/// [`Numeric`]: numeric/struct.Numeric.html
+/// [`Decimal`]: numeric/struct.Decimal.html
+/// [`BigDecimal`]: numeric/struct.BigDecimal.html
+/// [`XmlData`]: xml/struct.XmlData.html
+/// [`NaiveDateTime`]: time/chrono/struct.NaiveDateTime.html
+/// [`NaiveDate`]: time/chrono/struct.NaiveDate.html
+/// [`NaiveTime`]: time/chrono/struct.NaiveTime.html
+/// [`DateTime`]: time/chrono/struct.DateTime.html
+impl<'a> tiberius::IntoSql<'a> for ScalarImplTiberiusWrapper {
+    fn into_sql(self) -> tiberius::ColumnData<'a> {
+        match self.0 {
+            ScalarImpl::Int16(v) => v.into_sql(),
+            ScalarImpl::Int32(v) => v.into_sql(),
+            ScalarImpl::Int64(v) => v.into_sql(),
+            ScalarImpl::Float32(v) => v.0.into_sql(),
+            ScalarImpl::Float64(v) => v.0.into_sql(),
+            ScalarImpl::Bool(v) => v.into_sql(),
+            ScalarImpl::Decimal(v) => DecimalTiberiusWrapper::from(v).into_sql(),
+            ScalarImpl::Date(v) => DateTiberiusWrapper::from(v).into_sql(),
+            ScalarImpl::Timestamp(v) => TimestampTiberiusWrapper::from(v).into_sql(),
+            ScalarImpl::Timestamptz(v) => TimestamptzTiberiusWrapper::from(v).into_sql(),
+            ScalarImpl::Time(v) => TimeTiberiusWrapper::from(v).into_sql(),
+            // ScalarImpl::Bytea(v) => (*v.clone()).into_sql(),
+            value => {
+                // Utf8, Serial, Interval, Jsonb, Int256, Struct, List are not supported yet
+                unimplemented!("the sql server decoding for {:?} is unsupported", value);
+            }
+        }
+    }
+}
diff --git a/src/connector/src/parser/unified/debezium.rs b/src/connector/src/parser/unified/debezium.rs
index d90463698577d..5b9e6e1c1e720 100644
--- a/src/connector/src/parser/unified/debezium.rs
+++ b/src/connector/src/parser/unified/debezium.rs
@@ -12,14 +12,20 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use itertools::Itertools;
+use risingwave_common::catalog::{ColumnCatalog, ColumnDesc, ColumnId};
 use risingwave_common::types::{
     DataType, Datum, DatumCow, Scalar, ScalarImpl, ScalarRefImpl, Timestamptz, ToDatumRef,
 };
 use risingwave_connector_codec::decoder::AccessExt;
 use risingwave_pb::plan_common::additional_column::ColumnType;
+use thiserror_ext::AsReport;
 
 use super::{Access, AccessError, AccessResult, ChangeEvent, ChangeEventOperation};
+use crate::parser::debezium::schema_change::{SchemaChangeEnvelope, TableSchemaChange};
+use crate::parser::schema_change::TableChangeType;
 use crate::parser::TransactionControl;
+use crate::source::cdc::external::mysql::{mysql_type_to_rw_type, type_name_to_mysql_type};
 use crate::source::{ConnectorProperties, SourceColumnDesc};
 
 // Example of Debezium JSON value:
@@ -75,6 +81,8 @@ const OP: &str = "op";
 pub const TRANSACTION_STATUS: &str = "status";
 pub const TRANSACTION_ID: &str = "id";
 
+pub const TABLE_CHANGES: &str = "tableChanges";
+
 pub const DEBEZIUM_READ_OP: &str = "r";
 pub const DEBEZIUM_CREATE_OP: &str = "c";
 pub const DEBEZIUM_UPDATE_OP: &str = "u";
@@ -98,6 +106,7 @@ pub fn parse_transaction_meta(
         // The id field has different meanings for different databases:
         // PG: txID:LSN
         // MySQL: source_id:transaction_id (e.g. 3E11FA47-71CA-11E1-9E33-C80AA9429562:23)
+        // SQL Server: commit_lsn (e.g. 00000027:00000ac0:0002)
         match status {
             DEBEZIUM_TRANSACTION_STATUS_BEGIN => match *connector_props {
                 ConnectorProperties::PostgresCdc(_) => {
@@ -107,6 +116,9 @@ pub fn parse_transaction_meta(
                 ConnectorProperties::MysqlCdc(_) => {
                     return Ok(TransactionControl::Begin { id: id.into() })
                 }
+                ConnectorProperties::SqlServerCdc(_) => {
+                    return Ok(TransactionControl::Begin { id: id.into() })
+                }
                 _ => {}
             },
             DEBEZIUM_TRANSACTION_STATUS_COMMIT => match *connector_props {
@@ -117,6 +129,9 @@ pub fn parse_transaction_meta(
                 ConnectorProperties::MysqlCdc(_) => {
                     return Ok(TransactionControl::Commit { id: id.into() })
                 }
+                ConnectorProperties::SqlServerCdc(_) => {
+                    return Ok(TransactionControl::Commit { id: id.into() })
+                }
                 _ => {}
             },
             _ => {}
@@ -129,6 +144,96 @@ pub fn parse_transaction_meta(
     })
 }
 
+macro_rules! jsonb_access_field {
+    ($col:expr, $field:expr, $as_type:tt) => {
+        $crate::paste! {
+            $col.access_object_field($field).unwrap().[<as_ $as_type>]().unwrap()
+        }
+    };
+}
+
+pub fn parse_schema_change(
+    accessor: &impl Access,
+    connector_props: &ConnectorProperties,
+) -> AccessResult<SchemaChangeEnvelope> {
+    let mut schema_changes = vec![];
+
+    if let Some(ScalarRefImpl::List(table_changes)) = accessor
+        .access(&[TABLE_CHANGES], &DataType::List(Box::new(DataType::Jsonb)))?
+        .to_datum_ref()
+    {
+        for datum in table_changes.iter() {
+            let jsonb = match datum {
+                Some(ScalarRefImpl::Jsonb(jsonb)) => jsonb,
+                _ => unreachable!(""),
+            };
+
+            let id = jsonb_access_field!(jsonb, "id", string);
+            let ty = jsonb_access_field!(jsonb, "type", string);
+            let ddl_type: TableChangeType = ty.as_str().into();
+            if matches!(ddl_type, TableChangeType::Create | TableChangeType::Drop) {
+                tracing::debug!("skip table schema change for create/drop command");
+                continue;
+            }
+
+            let mut column_descs: Vec<ColumnDesc> = vec![];
+            if let Some(table) = jsonb.access_object_field("table")
+                && let Some(columns) = table.access_object_field("columns")
+            {
+                for col in columns.array_elements().unwrap() {
+                    let name = jsonb_access_field!(col, "name", string);
+                    let type_name = jsonb_access_field!(col, "typeName", string);
+
+                    let data_type = match *connector_props {
+                        ConnectorProperties::PostgresCdc(_) => {
+                            unimplemented!()
+                        }
+                        ConnectorProperties::MysqlCdc(_) => {
+                            let ty = type_name_to_mysql_type(type_name.as_str());
+                            match ty {
+                                Some(ty) => mysql_type_to_rw_type(&ty).map_err(|err| {
+                                    tracing::warn!(error=%err.as_report(), "unsupported mysql type in schema change message");
+                                    AccessError::UnsupportedType {
+                                        ty: type_name.clone(),
+                                    }
+                                })?,
+                                None => {
+                                    Err(AccessError::UnsupportedType { ty: type_name })?
+                                }
+                            }
+                        }
+                        _ => {
+                            unreachable!()
+                        }
+                    };
+
+                    column_descs.push(ColumnDesc::named(name, ColumnId::placeholder(), data_type));
+                }
+            }
+            schema_changes.push(TableSchemaChange {
+                cdc_table_name: id.replace('"', ""), // remove the double quotes
+                columns: column_descs
+                    .into_iter()
+                    .map(|column_desc| ColumnCatalog {
+                        column_desc,
+                        is_hidden: false,
+                    })
+                    .collect_vec(),
+                change_type: ty.as_str().into(),
+            });
+        }
+
+        Ok(SchemaChangeEnvelope {
+            table_changes: schema_changes,
+        })
+    } else {
+        Err(AccessError::Undefined {
+            name: "table schema change".into(),
+            path: TABLE_CHANGES.into(),
+        })
+    }
+}
+
 impl<A> DebeziumChangeEvent<A>
 where
     A: Access,
diff --git a/src/connector/src/parser/unified/json.rs b/src/connector/src/parser/unified/json.rs
index e4a229bb61b98..ca709e2eebc73 100644
--- a/src/connector/src/parser/unified/json.rs
+++ b/src/connector/src/parser/unified/json.rs
@@ -534,7 +534,7 @@ impl JsonParseOptions {
             (DataType::Struct(struct_type_info), ValueType::Object) => {
                 // Collecting into a Result<Vec<_>> doesn't reserve the capacity in advance, so we `Vec::with_capacity` instead.
                 // https://github.com/rust-lang/rust/issues/48994
-                let mut fields = Vec::with_capacity(struct_type_info.types().len());
+                let mut fields = Vec::with_capacity(struct_type_info.len());
                 for (field_name, field_type) in struct_type_info
                     .names()
                     .zip_eq_fast(struct_type_info.types())
diff --git a/src/connector/src/parser/upsert_parser.rs b/src/connector/src/parser/upsert_parser.rs
index df5e3b66e3136..c7bcce9f86a86 100644
--- a/src/connector/src/parser/upsert_parser.rs
+++ b/src/connector/src/parser/upsert_parser.rs
@@ -18,7 +18,7 @@ use risingwave_pb::plan_common::additional_column::ColumnType as AdditionalColum
 use super::bytes_parser::BytesAccessBuilder;
 use super::unified::{AccessImpl, ChangeEventOperation};
 use super::{
-    AccessBuilderImpl, ByteStreamSourceParser, BytesProperties, EncodingProperties, EncodingType,
+    AccessBuilderImpl, ByteStreamSourceParser, BytesProperties, EncodingProperties,
     SourceStreamChunkRowWriter, SpecificParserConfig,
 };
 use crate::error::ConnectorResult;
@@ -34,16 +34,11 @@ pub struct UpsertParser {
     source_ctx: SourceContextRef,
 }
 
-async fn build_accessor_builder(
-    config: EncodingProperties,
-    encoding_type: EncodingType,
-) -> ConnectorResult<AccessBuilderImpl> {
+async fn build_accessor_builder(config: EncodingProperties) -> ConnectorResult<AccessBuilderImpl> {
     match config {
         EncodingProperties::Json(_)
         | EncodingProperties::Protobuf(_)
-        | EncodingProperties::Avro(_) => {
-            Ok(AccessBuilderImpl::new_default(config, encoding_type).await?)
-        }
+        | EncodingProperties::Avro(_) => Ok(AccessBuilderImpl::new_default(config).await?),
         _ => bail!("unsupported encoding for Upsert"),
     }
 }
@@ -80,8 +75,7 @@ impl UpsertParser {
         } else {
             unreachable!("format upsert must have key column")
         };
-        let payload_builder =
-            build_accessor_builder(props.encoding_config, EncodingType::Value).await?;
+        let payload_builder = build_accessor_builder(props.encoding_config).await?;
         Ok(Self {
             key_builder,
             payload_builder,
diff --git a/src/connector/src/schema/mod.rs b/src/connector/src/schema/mod.rs
index 585dd43fa8bf1..9b3757e29c094 100644
--- a/src/connector/src/schema/mod.rs
+++ b/src/connector/src/schema/mod.rs
@@ -26,6 +26,7 @@ const KEY_MESSAGE_NAME_KEY: &str = "key.message";
 const SCHEMA_LOCATION_KEY: &str = "schema.location";
 const SCHEMA_REGISTRY_KEY: &str = "schema.registry";
 const NAME_STRATEGY_KEY: &str = "schema.registry.name.strategy";
+pub const AWS_GLUE_SCHEMA_ARN_KEY: &str = "aws.glue.schema_arn";
 
 #[derive(Debug, thiserror::Error, thiserror_ext::Macro)]
 #[error("Invalid option: {message}")]
diff --git a/src/connector/src/schema/schema_registry/util.rs b/src/connector/src/schema/schema_registry/util.rs
index 44b7a350e6823..a02d1deaf390d 100644
--- a/src/connector/src/schema/schema_registry/util.rs
+++ b/src/connector/src/schema/schema_registry/util.rs
@@ -53,11 +53,15 @@ pub enum WireFormatError {
     ParseMessageIndexes,
 }
 
-/// extract the magic number and `schema_id` at the front of payload
+/// Returns `(schema_id, payload)`
 ///
-/// 0 -> magic number
-/// 1-4 -> schema id
-/// 5-... -> message payload
+/// Refer to [Confluent schema registry wire format](https://docs.confluent.io/platform/7.6/schema-registry/fundamentals/serdes-develop/index.html#wire-format)
+///
+/// | Bytes | Area        | Description                                                                                        |
+/// |-------|-------------|----------------------------------------------------------------------------------------------------|
+/// | 0     | Magic Byte  | Confluent serialization format version number; currently always `0`.                               |
+/// | 1-4   | Schema ID   | 4-byte schema ID as returned by Schema Registry.                                                   |
+/// | 5-... | Data        | Serialized data for the specified schema format (for example, binary encoding for Avro or Protobuf.|
 pub(crate) fn extract_schema_id(payload: &[u8]) -> Result<(i32, &[u8]), WireFormatError> {
     use byteorder::{BigEndian, ReadBytesExt as _};
 
diff --git a/src/connector/src/sink/big_query.rs b/src/connector/src/sink/big_query.rs
index df800f234831c..bbe0139caaa46 100644
--- a/src/connector/src/sink/big_query.rs
+++ b/src/connector/src/sink/big_query.rs
@@ -16,7 +16,7 @@ use core::time::Duration;
 use std::collections::{BTreeMap, HashMap};
 use std::sync::Arc;
 
-use anyhow::anyhow;
+use anyhow::{anyhow, Context};
 use async_trait::async_trait;
 use gcp_bigquery_client::error::BQError;
 use gcp_bigquery_client::model::query_request::QueryRequest;
@@ -42,7 +42,7 @@ use prost_types::{
     FileDescriptorSet,
 };
 use risingwave_common::array::{Op, StreamChunk};
-use risingwave_common::buffer::Bitmap;
+use risingwave_common::bitmap::Bitmap;
 use risingwave_common::catalog::{Field, Schema};
 use risingwave_common::types::DataType;
 use serde_derive::Deserialize;
@@ -211,14 +211,14 @@ impl BigQuerySink {
         for i in rw_fields_name {
             let value = big_query_columns_desc.get(&i.name).ok_or_else(|| {
                 SinkError::BigQuery(anyhow::anyhow!(
-                    "Column name don't find in bigquery, risingwave is {:?} ",
+                    "Column `{:?}` on RisingWave side is not found on BigQuery side.",
                     i.name
                 ))
             })?;
             let data_type_string = Self::get_string_and_check_support_from_datatype(&i.data_type)?;
             if data_type_string.ne(value) {
                 return Err(SinkError::BigQuery(anyhow::anyhow!(
-                    "Column type don't match, column name is {:?}. bigquery type is {:?} risingwave type is {:?} ",i.name,value,data_type_string
+                    "Data type mismatch for column `{:?}`. BigQuery side: `{:?}`, RisingWave side: `{:?}`. ", i.name, value, data_type_string
                 )));
             };
         }
@@ -489,7 +489,7 @@ impl BigQuerySinkWriter {
             descriptor_proto.field.push(field);
         }
 
-        let descriptor_pool = build_protobuf_descriptor_pool(&descriptor_proto);
+        let descriptor_pool = build_protobuf_descriptor_pool(&descriptor_proto)?;
         let message_descriptor = descriptor_pool
             .get_message_by_name(&config.common.table)
             .ok_or_else(|| {
@@ -733,7 +733,7 @@ impl StorageWriterClient {
     }
 }
 
-fn build_protobuf_descriptor_pool(desc: &DescriptorProto) -> prost_reflect::DescriptorPool {
+fn build_protobuf_descriptor_pool(desc: &DescriptorProto) -> Result<prost_reflect::DescriptorPool> {
     let file_descriptor = FileDescriptorProto {
         message_type: vec![desc.clone()],
         name: Some("bigquery".to_string()),
@@ -743,7 +743,8 @@ fn build_protobuf_descriptor_pool(desc: &DescriptorProto) -> prost_reflect::Desc
     prost_reflect::DescriptorPool::from_file_descriptor_set(FileDescriptorSet {
         file: vec![file_descriptor],
     })
-    .unwrap()
+    .context("failed to build descriptor pool")
+    .map_err(SinkError::BigQuery)
 }
 
 fn build_protobuf_schema<'a>(
@@ -876,7 +877,7 @@ mod test {
             .iter()
             .map(|f| (f.name.as_str(), &f.data_type));
         let desc = build_protobuf_schema(fields, "t1".to_string()).unwrap();
-        let pool = build_protobuf_descriptor_pool(&desc);
+        let pool = build_protobuf_descriptor_pool(&desc).unwrap();
         let t1_message = pool.get_message_by_name("t1").unwrap();
         assert_matches!(
             t1_message.get_field_by_name("v1").unwrap().kind(),
diff --git a/src/connector/src/sink/boxed.rs b/src/connector/src/sink/boxed.rs
index bb3edaa22f087..f31e3b0a06805 100644
--- a/src/connector/src/sink/boxed.rs
+++ b/src/connector/src/sink/boxed.rs
@@ -17,7 +17,7 @@ use std::sync::Arc;
 
 use async_trait::async_trait;
 use risingwave_common::array::StreamChunk;
-use risingwave_common::buffer::Bitmap;
+use risingwave_common::bitmap::Bitmap;
 use risingwave_pb::connector_service::SinkMetadata;
 
 use crate::sink::{SinkCommitCoordinator, SinkWriter};
diff --git a/src/connector/src/sink/catalog/desc.rs b/src/connector/src/sink/catalog/desc.rs
index 64e618bdd933e..cfd632170d3fb 100644
--- a/src/connector/src/sink/catalog/desc.rs
+++ b/src/connector/src/sink/catalog/desc.rs
@@ -51,6 +51,9 @@ pub struct SinkDesc {
     /// The properties of the sink.
     pub properties: BTreeMap<String, String>,
 
+    /// Secret ref
+    pub secret_refs: BTreeMap<String, PbSecretRef>,
+
     // The append-only behavior of the physical sink connector. Frontend will determine `sink_type`
     // based on both its own derivation on the append-only attribute and other user-specified
     // options in `properties`.
@@ -84,7 +87,6 @@ impl SinkDesc {
         owner: UserId,
         connection_id: Option<ConnectionId>,
         dependent_relations: Vec<TableId>,
-        secret_ref: BTreeMap<String, PbSecretRef>,
     ) -> SinkCatalog {
         SinkCatalog {
             id: self.id,
@@ -99,7 +101,7 @@ impl SinkDesc {
             owner,
             dependent_relations,
             properties: self.properties,
-            secret_refs: secret_ref,
+            secret_refs: self.secret_refs,
             sink_type: self.sink_type,
             format_desc: self.format_desc,
             connection_id,
@@ -111,6 +113,7 @@ impl SinkDesc {
             created_at_cluster_version: None,
             initialized_at_cluster_version: None,
             create_type: self.create_type,
+            original_target_columns: vec![],
         }
     }
 
@@ -134,7 +137,7 @@ impl SinkDesc {
             sink_from_name: self.sink_from_name.clone(),
             target_table: self.target_table.map(|table_id| table_id.table_id()),
             extra_partition_col_idx: self.extra_partition_col_idx.map(|idx| idx as u64),
-            secret_refs: Default::default(),
+            secret_refs: self.secret_refs.clone(),
         }
     }
 }
diff --git a/src/connector/src/sink/catalog/mod.rs b/src/connector/src/sink/catalog/mod.rs
index cb814154c8222..7a9dc5d564ca7 100644
--- a/src/connector/src/sink/catalog/mod.rs
+++ b/src/connector/src/sink/catalog/mod.rs
@@ -15,6 +15,7 @@
 pub mod desc;
 
 use std::collections::BTreeMap;
+use std::fmt::{Display, Formatter};
 
 use anyhow::anyhow;
 use itertools::Itertools;
@@ -28,6 +29,7 @@ use risingwave_pb::catalog::{
     PbCreateType, PbSink, PbSinkFormatDesc, PbSinkType, PbStreamJobStatus,
 };
 use risingwave_pb::secret::PbSecretRef;
+use serde_derive::Serialize;
 
 use super::{
     SinkError, CONNECTOR_TYPE_KEY, SINK_TYPE_APPEND_ONLY, SINK_TYPE_DEBEZIUM, SINK_TYPE_OPTION,
@@ -120,20 +122,26 @@ pub struct SinkFormatDesc {
     pub format: SinkFormat,
     pub encode: SinkEncode,
     pub options: BTreeMap<String, String>,
-
+    pub secret_refs: BTreeMap<String, PbSecretRef>,
     pub key_encode: Option<SinkEncode>,
 }
 
 /// TODO: consolidate with [`crate::source::SourceFormat`] and [`crate::parser::ProtocolProperties`].
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize)]
 pub enum SinkFormat {
     AppendOnly,
     Upsert,
     Debezium,
 }
 
+impl Display for SinkFormat {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{:?}", self)
+    }
+}
+
 /// TODO: consolidate with [`crate::source::SourceEncode`] and [`crate::parser::EncodingProperties`].
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize)]
 pub enum SinkEncode {
     Json,
     Protobuf,
@@ -142,6 +150,12 @@ pub enum SinkEncode {
     Text,
 }
 
+impl Display for SinkEncode {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{:?}", self)
+    }
+}
+
 impl SinkFormatDesc {
     pub fn from_legacy_type(connector: &str, r#type: &str) -> Result<Option<Self>, SinkError> {
         use crate::sink::kafka::KafkaSink;
@@ -170,6 +184,7 @@ impl SinkFormatDesc {
             format,
             encode,
             options: Default::default(),
+            secret_refs: Default::default(),
             key_encode: None,
         }))
     }
@@ -203,7 +218,7 @@ impl SinkFormatDesc {
             encode: encode.into(),
             options,
             key_encode,
-            secret_refs: Default::default(),
+            secret_refs: self.secret_refs.clone(),
         }
     }
 }
@@ -235,7 +250,13 @@ impl TryFrom<PbSinkFormatDesc> for SinkFormatDesc {
             E::Protobuf => SinkEncode::Protobuf,
             E::Template => SinkEncode::Template,
             E::Avro => SinkEncode::Avro,
-            e @ (E::Unspecified | E::Native | E::Csv | E::Bytes | E::None | E::Text) => {
+            e @ (E::Unspecified
+            | E::Native
+            | E::Csv
+            | E::Bytes
+            | E::None
+            | E::Text
+            | E::Parquet) => {
                 return Err(SinkError::Config(anyhow!(
                     "sink encode unsupported: {}",
                     e.as_str_name()
@@ -252,6 +273,7 @@ impl TryFrom<PbSinkFormatDesc> for SinkFormatDesc {
             | E::Protobuf
             | E::Template
             | E::Native
+            | E::Parquet
             | E::None) => {
                 return Err(SinkError::Config(anyhow!(
                     "unsupported {} as sink key encode",
@@ -259,13 +281,13 @@ impl TryFrom<PbSinkFormatDesc> for SinkFormatDesc {
                 )))
             }
         };
-        let options = value.options.into_iter().collect();
 
         Ok(Self {
             format,
             encode,
-            options,
+            options: value.options,
             key_encode,
+            secret_refs: value.secret_refs,
         })
     }
 }
@@ -342,6 +364,9 @@ pub struct SinkCatalog {
 
     /// The secret reference for the sink, mapping from property name to secret id.
     pub secret_refs: BTreeMap<String, PbSecretRef>,
+
+    /// Only for the sink whose target is a table. Columns of the target table when the sink is created. At this point all the default columns of the target table are all handled by the project operator in the sink plan.
+    pub original_target_columns: Vec<ColumnCatalog>,
 }
 
 impl SinkCatalog {
@@ -384,6 +409,11 @@ impl SinkCatalog {
             initialized_at_cluster_version: self.initialized_at_cluster_version.clone(),
             create_type: self.create_type.to_proto() as i32,
             secret_refs: self.secret_refs.clone(),
+            original_target_columns: self
+                .original_target_columns
+                .iter()
+                .map(|c| c.to_protobuf())
+                .collect_vec(),
         }
     }
 
@@ -420,6 +450,11 @@ impl SinkCatalog {
     pub fn downstream_pk_indices(&self) -> Vec<usize> {
         self.downstream_pk.clone()
     }
+
+    pub fn unique_identity(&self) -> String {
+        // We need to align with meta here, so we've utilized the proto method.
+        self.to_proto().unique_identity()
+    }
 }
 
 impl From<PbSink> for SinkCatalog {
@@ -478,6 +513,11 @@ impl From<PbSink> for SinkCatalog {
             created_at_cluster_version: pb.created_at_cluster_version,
             create_type: CreateType::from_proto(create_type),
             secret_refs: pb.secret_refs,
+            original_target_columns: pb
+                .original_target_columns
+                .into_iter()
+                .map(ColumnCatalog::from)
+                .collect_vec(),
         }
     }
 }
diff --git a/src/connector/src/sink/clickhouse.rs b/src/connector/src/sink/clickhouse.rs
index bd5ebe8a6ce8d..d715e93b8d6c4 100644
--- a/src/connector/src/sink/clickhouse.rs
+++ b/src/connector/src/sink/clickhouse.rs
@@ -22,7 +22,7 @@ use clickhouse::insert::Insert;
 use clickhouse::{Client as ClickHouseClient, Row as ClickHouseRow};
 use itertools::Itertools;
 use risingwave_common::array::{Op, StreamChunk};
-use risingwave_common::buffer::Bitmap;
+use risingwave_common::bitmap::Bitmap;
 use risingwave_common::catalog::Schema;
 use risingwave_common::row::Row;
 use risingwave_common::session_config::sink_decouple::SinkDecouple;
@@ -85,11 +85,17 @@ enum ClickHouseEngine {
     ReplicatedReplacingMergeTree(Option<String>),
     ReplicatedSummingMergeTree,
     ReplicatedAggregatingMergeTree,
-    #[expect(dead_code)]
     ReplicatedCollapsingMergeTree(String),
-    #[expect(dead_code)]
     ReplicatedVersionedCollapsingMergeTree(String),
     ReplicatedGraphiteMergeTree,
+    SharedMergeTree,
+    SharedReplacingMergeTree(Option<String>),
+    SharedSummingMergeTree,
+    SharedAggregatingMergeTree,
+    SharedCollapsingMergeTree(String),
+    SharedVersionedCollapsingMergeTree(String),
+    SharedGraphiteMergeTree,
+    Null,
 }
 impl ClickHouseEngine {
     pub fn is_collapsing_engine(&self) -> bool {
@@ -99,6 +105,8 @@ impl ClickHouseEngine {
                 | ClickHouseEngine::VersionedCollapsingMergeTree(_)
                 | ClickHouseEngine::ReplicatedCollapsingMergeTree(_)
                 | ClickHouseEngine::ReplicatedVersionedCollapsingMergeTree(_)
+                | ClickHouseEngine::SharedCollapsingMergeTree(_)
+                | ClickHouseEngine::SharedVersionedCollapsingMergeTree(_)
         )
     }
 
@@ -106,6 +114,7 @@ impl ClickHouseEngine {
         match self {
             ClickHouseEngine::ReplacingMergeTree(delete_col) => delete_col.is_some(),
             ClickHouseEngine::ReplicatedReplacingMergeTree(delete_col) => delete_col.is_some(),
+            ClickHouseEngine::SharedReplacingMergeTree(delete_col) => delete_col.is_some(),
             _ => false,
         }
     }
@@ -116,6 +125,9 @@ impl ClickHouseEngine {
             ClickHouseEngine::ReplicatedReplacingMergeTree(Some(delete_col)) => {
                 Some(delete_col.to_string())
             }
+            ClickHouseEngine::SharedReplacingMergeTree(Some(delete_col)) => {
+                Some(delete_col.to_string())
+            }
             _ => None,
         }
     }
@@ -132,16 +144,34 @@ impl ClickHouseEngine {
             ClickHouseEngine::ReplicatedVersionedCollapsingMergeTree(sign_name) => {
                 Some(sign_name.to_string())
             }
+            ClickHouseEngine::SharedCollapsingMergeTree(sign_name) => Some(sign_name.to_string()),
+            ClickHouseEngine::SharedVersionedCollapsingMergeTree(sign_name) => {
+                Some(sign_name.to_string())
+            }
             _ => None,
         }
     }
 
+    pub fn is_shared_tree(&self) -> bool {
+        matches!(
+            self,
+            ClickHouseEngine::SharedMergeTree
+                | ClickHouseEngine::SharedReplacingMergeTree(_)
+                | ClickHouseEngine::SharedSummingMergeTree
+                | ClickHouseEngine::SharedAggregatingMergeTree
+                | ClickHouseEngine::SharedCollapsingMergeTree(_)
+                | ClickHouseEngine::SharedVersionedCollapsingMergeTree(_)
+                | ClickHouseEngine::SharedGraphiteMergeTree
+        )
+    }
+
     pub fn from_query_engine(
         engine_name: &ClickhouseQueryEngine,
         config: &ClickHouseConfig,
     ) -> Result<Self> {
         match engine_name.engine.as_str() {
             "MergeTree" => Ok(ClickHouseEngine::MergeTree),
+            "Null" => Ok(ClickHouseEngine::Null),
             "ReplacingMergeTree" => {
                 let delete_column = config.common.delete_column.clone();
                 Ok(ClickHouseEngine::ReplacingMergeTree(delete_column))
@@ -201,7 +231,9 @@ impl ClickHouseEngine {
                     .ok_or_else(|| SinkError::ClickHouse("must have index 1".to_string()))?
                     .trim()
                     .to_string();
-                Ok(ClickHouseEngine::VersionedCollapsingMergeTree(sign_name))
+                Ok(ClickHouseEngine::ReplicatedVersionedCollapsingMergeTree(
+                    sign_name,
+                ))
             }
             // ReplicatedCollapsingMergeTree("a","b",sign_name)
             "ReplicatedCollapsingMergeTree" => {
@@ -218,9 +250,51 @@ impl ClickHouseEngine {
                     .ok_or_else(|| SinkError::ClickHouse("must have last".to_string()))?
                     .trim()
                     .to_string();
-                Ok(ClickHouseEngine::CollapsingMergeTree(sign_name))
+                Ok(ClickHouseEngine::ReplicatedCollapsingMergeTree(sign_name))
             }
             "ReplicatedGraphiteMergeTree" => Ok(ClickHouseEngine::ReplicatedGraphiteMergeTree),
+            "SharedMergeTree" => Ok(ClickHouseEngine::SharedMergeTree),
+            "SharedReplacingMergeTree" => {
+                let delete_column = config.common.delete_column.clone();
+                Ok(ClickHouseEngine::SharedReplacingMergeTree(delete_column))
+            }
+            "SharedSummingMergeTree" => Ok(ClickHouseEngine::SharedSummingMergeTree),
+            "SharedAggregatingMergeTree" => Ok(ClickHouseEngine::SharedAggregatingMergeTree),
+            // SharedVersionedCollapsingMergeTree("a","b",sign_name,"c")
+            "SharedVersionedCollapsingMergeTree" => {
+                let sign_name = engine_name
+                    .create_table_query
+                    .split("SharedVersionedCollapsingMergeTree(")
+                    .last()
+                    .ok_or_else(|| SinkError::ClickHouse("must have last".to_string()))?
+                    .split(',')
+                    .rev()
+                    .nth(1)
+                    .ok_or_else(|| SinkError::ClickHouse("must have index 1".to_string()))?
+                    .trim()
+                    .to_string();
+                Ok(ClickHouseEngine::SharedVersionedCollapsingMergeTree(
+                    sign_name,
+                ))
+            }
+            // SharedCollapsingMergeTree("a","b",sign_name)
+            "SharedCollapsingMergeTree" => {
+                let sign_name = engine_name
+                    .create_table_query
+                    .split("SharedCollapsingMergeTree(")
+                    .last()
+                    .ok_or_else(|| SinkError::ClickHouse("must have last".to_string()))?
+                    .split(')')
+                    .next()
+                    .ok_or_else(|| SinkError::ClickHouse("must have next".to_string()))?
+                    .split(',')
+                    .last()
+                    .ok_or_else(|| SinkError::ClickHouse("must have last".to_string()))?
+                    .trim()
+                    .to_string();
+                Ok(ClickHouseEngine::SharedCollapsingMergeTree(sign_name))
+            }
+            "SharedGraphiteMergeTree" => Ok(ClickHouseEngine::SharedGraphiteMergeTree),
             _ => Err(SinkError::ClickHouse(format!(
                 "Cannot find clickhouse engine {:?}",
                 engine_name.engine
@@ -452,13 +526,18 @@ impl Sink for ClickHouseSink {
 
         let (clickhouse_column, clickhouse_engine) =
             query_column_engine_from_ck(client, &self.config).await?;
+        if clickhouse_engine.is_shared_tree() {
+            risingwave_common::license::Feature::ClickHouseSharedEngine
+                .check_available()
+                .map_err(|e| anyhow::anyhow!(e))?;
+        }
 
         if !self.is_append_only
             && !clickhouse_engine.is_collapsing_engine()
             && !clickhouse_engine.is_delete_replacing_engine()
         {
             return match clickhouse_engine {
-                ClickHouseEngine::ReplicatedReplacingMergeTree(None) | ClickHouseEngine::ReplacingMergeTree(None) =>  {
+                ClickHouseEngine::ReplicatedReplacingMergeTree(None) | ClickHouseEngine::ReplacingMergeTree(None) | ClickHouseEngine::SharedReplacingMergeTree(None) =>  {
                     Err(SinkError::ClickHouse("To enable upsert with a `ReplacingMergeTree`, you must set a `clickhouse.delete.column` to the UInt8 column in ClickHouse used to signify deletes. See https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/replacingmergetree#is_deleted for more information".to_owned()))
                 }
                 _ => Err(SinkError::ClickHouse("If you want to use upsert, please use either `VersionedCollapsingMergeTree`, `CollapsingMergeTree` or the `ReplacingMergeTree` in ClickHouse".to_owned()))
@@ -823,7 +902,7 @@ impl ClickHouseFieldWithNull {
     ) -> Result<Vec<ClickHouseFieldWithNull>> {
         let clickhouse_schema_feature = clickhouse_schema_feature_vec
             .get(clickhouse_schema_feature_index)
-            .unwrap();
+            .ok_or_else(|| SinkError::ClickHouse(format!("No column found from clickhouse table schema, index is {clickhouse_schema_feature_index}")))?;
         if data.is_none() {
             if !clickhouse_schema_feature.can_null {
                 return Err(SinkError::ClickHouse(
@@ -1011,7 +1090,7 @@ pub fn build_fields_name_type_from_schema(schema: &Schema) -> Result<Vec<(String
             for i in &field.sub_fields {
                 if matches!(i.data_type, DataType::Struct(_)) {
                     return Err(SinkError::ClickHouse(
-                        "Only one level of nesting is supported for sturct".to_string(),
+                        "Only one level of nesting is supported for struct".to_string(),
                     ));
                 } else {
                     vec.push((
diff --git a/src/connector/src/sink/coordinate.rs b/src/connector/src/sink/coordinate.rs
index f2bc4f4b3b362..c069167870101 100644
--- a/src/connector/src/sink/coordinate.rs
+++ b/src/connector/src/sink/coordinate.rs
@@ -16,7 +16,7 @@ use std::sync::Arc;
 
 use anyhow::anyhow;
 use risingwave_common::array::StreamChunk;
-use risingwave_common::buffer::Bitmap;
+use risingwave_common::bitmap::Bitmap;
 use risingwave_pb::connector_service::SinkMetadata;
 use risingwave_rpc_client::CoordinatorStreamHandle;
 use tracing::warn;
diff --git a/src/connector/src/sink/deltalake.rs b/src/connector/src/sink/deltalake.rs
index 4ad98da4975fa..661145959a0ff 100644
--- a/src/connector/src/sink/deltalake.rs
+++ b/src/connector/src/sink/deltalake.rs
@@ -29,7 +29,7 @@ use deltalake::DeltaTable;
 use risingwave_common::array::arrow::DeltaLakeConvert;
 use risingwave_common::array::StreamChunk;
 use risingwave_common::bail;
-use risingwave_common::buffer::Bitmap;
+use risingwave_common::bitmap::Bitmap;
 use risingwave_common::catalog::Schema;
 use risingwave_common::session_config::sink_decouple::SinkDecouple;
 use risingwave_common::types::DataType;
@@ -135,7 +135,7 @@ impl DeltaLakeCommon {
             Ok(DeltaTableUrl::Local(path.to_string()))
         } else {
             Err(SinkError::DeltaLake(anyhow!(
-                "path need to start with 's3://','s3a://'(s3) ,gs://(gcs) or file://(local)"
+                "path should start with 's3://','s3a://'(s3) ,gs://(gcs) or file://(local)"
             )))
         }
     }
@@ -356,7 +356,7 @@ impl Sink for DeltaLakeSink {
             .collect();
         if deltalake_fields.len() != self.param.schema().fields().len() {
             return Err(SinkError::DeltaLake(anyhow!(
-                "column count not match, rw is {}, deltalake is {}",
+                "Columns mismatch. RisingWave is {}, DeltaLake is {}",
                 self.param.schema().fields().len(),
                 deltalake_fields.len()
             )));
diff --git a/src/connector/src/sink/doris.rs b/src/connector/src/sink/doris.rs
index f8c7b374d11ce..7745f2e9e98b1 100644
--- a/src/connector/src/sink/doris.rs
+++ b/src/connector/src/sink/doris.rs
@@ -21,7 +21,7 @@ use base64::engine::general_purpose;
 use base64::Engine;
 use bytes::{BufMut, Bytes, BytesMut};
 use risingwave_common::array::{Op, StreamChunk};
-use risingwave_common::buffer::Bitmap;
+use risingwave_common::bitmap::Bitmap;
 use risingwave_common::catalog::Schema;
 use risingwave_common::types::DataType;
 use serde::Deserialize;
@@ -268,11 +268,11 @@ impl DorisSinkWriter {
             .build_get_client()
             .get_schema_from_doris()
             .await?;
-        doris_schema.properties.iter().for_each(|s| {
-            if let Some(v) = s.get_decimal_pre_scale() {
+        for s in &doris_schema.properties {
+            if let Some(v) = s.get_decimal_pre_scale()? {
                 decimal_map.insert(s.name.clone(), v);
             }
-        });
+        }
 
         let header_builder = HeaderBuilder::new()
             .add_common_header()
@@ -491,11 +491,27 @@ pub struct DorisField {
     aggregation_type: String,
 }
 impl DorisField {
-    pub fn get_decimal_pre_scale(&self) -> Option<u8> {
+    pub fn get_decimal_pre_scale(&self) -> Result<Option<u8>> {
         if self.r#type.contains("DECIMAL") {
-            Some(self.scale.clone().unwrap().parse::<u8>().unwrap())
+            let scale = self
+                .scale
+                .as_ref()
+                .ok_or_else(|| {
+                    SinkError::Doris(format!(
+                        "In doris, the type of {} is DECIMAL, but `scale` is not found",
+                        self.name
+                    ))
+                })?
+                .parse::<u8>()
+                .map_err(|err| {
+                    SinkError::Doris(format!(
+                        "Unable to convert decimal's scale to u8. error: {:?}",
+                        err.kind()
+                    ))
+                })?;
+            Ok(Some(scale))
         } else {
-            None
+            Ok(None)
         }
     }
 }
diff --git a/src/connector/src/sink/doris_starrocks_connector.rs b/src/connector/src/sink/doris_starrocks_connector.rs
index fb0a37572710e..062b6f28bb226 100644
--- a/src/connector/src/sink/doris_starrocks_connector.rs
+++ b/src/connector/src/sink/doris_starrocks_connector.rs
@@ -256,22 +256,22 @@ impl InserterInnerBuilder {
         })
     }
 
-    fn build_request(&self, uri: String) -> RequestBuilder {
+    fn build_request(&self, uri: String) -> Result<RequestBuilder> {
         let client = Client::builder()
             .pool_idle_timeout(POOL_IDLE_TIMEOUT)
             .redirect(redirect::Policy::none()) // we handle redirect by ourselves
             .build()
-            .unwrap();
+            .map_err(|err| SinkError::DorisStarrocksConnect(anyhow!(err)))?;
 
         let mut builder = client.put(uri);
         for (k, v) in &self.header {
             builder = builder.header(k, v);
         }
-        builder
+        Ok(builder)
     }
 
     pub async fn build(&self) -> Result<InserterInner> {
-        let builder = self.build_request(self.url.clone());
+        let builder = self.build_request(self.url.clone())?;
         let resp = builder
             .send()
             .await
@@ -284,7 +284,7 @@ impl InserterInnerBuilder {
         let body = Body::wrap_stream(
             tokio_stream::wrappers::UnboundedReceiverStream::new(receiver).map(Ok::<_, Infallible>),
         );
-        let builder = self.build_request(be_url.into()).body(body);
+        let builder = self.build_request(be_url.into())?.body(body);
 
         let handle: JoinHandle<Result<Vec<u8>>> = tokio::spawn(async move {
             let response = builder
@@ -321,7 +321,7 @@ type Sender = UnboundedSender<Bytes>;
 
 pub struct InserterInner {
     sender: Option<Sender>,
-    join_handle: Option<JoinHandle<Result<Vec<u8>>>>,
+    join_handle: JoinHandle<Result<Vec<u8>>>,
     buffer: BytesMut,
     stream_load_http_timeout: Duration,
 }
@@ -333,7 +333,7 @@ impl InserterInner {
     ) -> Self {
         Self {
             sender: Some(sender),
-            join_handle: Some(join_handle),
+            join_handle,
             buffer: BytesMut::with_capacity(BUFFER_SIZE),
             stream_load_http_timeout,
         }
@@ -365,11 +365,8 @@ impl InserterInner {
     }
 
     async fn wait_handle(&mut self) -> Result<Vec<u8>> {
-        let res = match tokio::time::timeout(
-            self.stream_load_http_timeout,
-            self.join_handle.as_mut().unwrap(),
-        )
-        .await
+        let res = match tokio::time::timeout(self.stream_load_http_timeout, &mut self.join_handle)
+            .await
         {
             Ok(res) => res.map_err(|err| SinkError::DorisStarrocksConnect(anyhow!(err)))??,
             Err(err) => return Err(SinkError::DorisStarrocksConnect(anyhow!(err))),
@@ -480,7 +477,7 @@ impl StarrocksTxnRequestBuilder {
             .pool_idle_timeout(POOL_IDLE_TIMEOUT)
             .redirect(redirect::Policy::none())
             .build()
-            .unwrap();
+            .map_err(|err| SinkError::DorisStarrocksConnect(anyhow!(err)))?;
 
         Ok(Self {
             url_begin,
diff --git a/src/connector/src/sink/elasticsearch.rs b/src/connector/src/sink/elasticsearch.rs
index 3d51e48201c94..31dde5c52509e 100644
--- a/src/connector/src/sink/elasticsearch.rs
+++ b/src/connector/src/sink/elasticsearch.rs
@@ -87,7 +87,7 @@ impl EsStreamChunkConverter {
             Box::new(|row: RowRef<'_>| {
                 Ok(row
                     .datum_at(0)
-                    .ok_or_else(|| anyhow!("No value find in row, index is 0"))?
+                    .ok_or_else(|| anyhow!("No value found in row, index is 0"))?
                     .to_text())
             })
         } else if pk_indices.len() == 1 {
@@ -95,7 +95,7 @@ impl EsStreamChunkConverter {
             Box::new(move |row: RowRef<'_>| {
                 Ok(row
                     .datum_at(index)
-                    .ok_or_else(|| anyhow!("No value find in row, index is 0"))?
+                    .ok_or_else(|| anyhow!("No value found in row, index is 0"))?
                     .to_text())
             })
         } else {
@@ -108,7 +108,7 @@ impl EsStreamChunkConverter {
                 for index in &pk_indices {
                     keys.push(
                         row.datum_at(*index)
-                            .ok_or_else(|| anyhow!("No value find in row, index is {}", index))?
+                            .ok_or_else(|| anyhow!("No value found in row, index is {}", index))?
                             .to_text(),
                     );
                 }
@@ -144,7 +144,7 @@ impl EsStreamChunkConverter {
             if let Some(index) = self.index_column {
                 index_builder.append(Some(
                     row.datum_at(index)
-                        .ok_or_else(|| anyhow!("No value find in row, index is {}", index))?
+                        .ok_or_else(|| anyhow!("No value found in row, index is {}", index))?
                         .into_utf8(),
                 ));
             } else {
diff --git a/src/connector/src/sink/encoder/json.rs b/src/connector/src/sink/encoder/json.rs
index 4748116609700..3652f38bacbb2 100644
--- a/src/connector/src/sink/encoder/json.rs
+++ b/src/connector/src/sink/encoder/json.rs
@@ -30,20 +30,25 @@ use serde_json::{json, Map, Value};
 use thiserror_ext::AsReport;
 
 use super::{
-    CustomJsonType, DateHandlingMode, KafkaConnectParams, KafkaConnectParamsRef, Result,
-    RowEncoder, SerTo, TimeHandlingMode, TimestampHandlingMode, TimestamptzHandlingMode,
+    CustomJsonType, DateHandlingMode, JsonbHandlingMode, KafkaConnectParams, KafkaConnectParamsRef,
+    Result, RowEncoder, SerTo, TimeHandlingMode, TimestampHandlingMode, TimestamptzHandlingMode,
 };
 use crate::sink::SinkError;
 
-pub struct JsonEncoder {
-    schema: Schema,
-    col_indices: Option<Vec<usize>>,
+pub struct JsonEncoderConfig {
     time_handling_mode: TimeHandlingMode,
     date_handling_mode: DateHandlingMode,
     timestamp_handling_mode: TimestampHandlingMode,
     timestamptz_handling_mode: TimestamptzHandlingMode,
     custom_json_type: CustomJsonType,
+    jsonb_handling_mode: JsonbHandlingMode,
+}
+
+pub struct JsonEncoder {
+    schema: Schema,
+    col_indices: Option<Vec<usize>>,
     kafka_connect: Option<KafkaConnectParamsRef>,
+    config: JsonEncoderConfig,
 }
 
 impl JsonEncoder {
@@ -54,29 +59,38 @@ impl JsonEncoder {
         timestamp_handling_mode: TimestampHandlingMode,
         timestamptz_handling_mode: TimestamptzHandlingMode,
         time_handling_mode: TimeHandlingMode,
+        jsonb_handling_mode: JsonbHandlingMode,
     ) -> Self {
-        Self {
-            schema,
-            col_indices,
+        let config = JsonEncoderConfig {
             time_handling_mode,
             date_handling_mode,
             timestamp_handling_mode,
             timestamptz_handling_mode,
             custom_json_type: CustomJsonType::None,
+            jsonb_handling_mode,
+        };
+        Self {
+            schema,
+            col_indices,
             kafka_connect: None,
+            config,
         }
     }
 
     pub fn new_with_es(schema: Schema, col_indices: Option<Vec<usize>>) -> Self {
-        Self {
-            schema,
-            col_indices,
+        let config = JsonEncoderConfig {
             time_handling_mode: TimeHandlingMode::String,
             date_handling_mode: DateHandlingMode::String,
             timestamp_handling_mode: TimestampHandlingMode::String,
             timestamptz_handling_mode: TimestamptzHandlingMode::UtcWithoutSuffix,
             custom_json_type: CustomJsonType::Es,
+            jsonb_handling_mode: JsonbHandlingMode::Dynamic,
+        };
+        Self {
+            schema,
+            col_indices,
             kafka_connect: None,
+            config,
         }
     }
 
@@ -85,28 +99,36 @@ impl JsonEncoder {
         col_indices: Option<Vec<usize>>,
         map: HashMap<String, u8>,
     ) -> Self {
-        Self {
-            schema,
-            col_indices,
+        let config = JsonEncoderConfig {
             time_handling_mode: TimeHandlingMode::Milli,
             date_handling_mode: DateHandlingMode::String,
             timestamp_handling_mode: TimestampHandlingMode::String,
             timestamptz_handling_mode: TimestamptzHandlingMode::UtcWithoutSuffix,
             custom_json_type: CustomJsonType::Doris(map),
+            jsonb_handling_mode: JsonbHandlingMode::String,
+        };
+        Self {
+            schema,
+            col_indices,
             kafka_connect: None,
+            config,
         }
     }
 
     pub fn new_with_starrocks(schema: Schema, col_indices: Option<Vec<usize>>) -> Self {
-        Self {
-            schema,
-            col_indices,
+        let config = JsonEncoderConfig {
             time_handling_mode: TimeHandlingMode::Milli,
             date_handling_mode: DateHandlingMode::String,
             timestamp_handling_mode: TimestampHandlingMode::String,
             timestamptz_handling_mode: TimestamptzHandlingMode::UtcWithoutSuffix,
             custom_json_type: CustomJsonType::StarRocks,
+            jsonb_handling_mode: JsonbHandlingMode::Dynamic,
+        };
+        Self {
+            schema,
+            col_indices,
             kafka_connect: None,
+            config,
         }
     }
 
@@ -139,16 +161,8 @@ impl RowEncoder for JsonEncoder {
         for idx in &col_indices {
             let field = &self.schema[*idx];
             let key = field.name.clone();
-            let value = datum_to_json_object(
-                field,
-                row.datum_at(*idx),
-                self.date_handling_mode,
-                self.timestamp_handling_mode,
-                self.timestamptz_handling_mode,
-                self.time_handling_mode,
-                &self.custom_json_type,
-            )
-            .map_err(|e| SinkError::Encode(e.to_report_string()))?;
+            let value = datum_to_json_object(field, row.datum_at(*idx), &self.config)
+                .map_err(|e| SinkError::Encode(e.to_report_string()))?;
             mappings.insert(key, value);
         }
 
@@ -179,11 +193,7 @@ impl SerTo<String> for Value {
 fn datum_to_json_object(
     field: &Field,
     datum: DatumRef<'_>,
-    date_handling_mode: DateHandlingMode,
-    timestamp_handling_mode: TimestampHandlingMode,
-    timestamptz_handling_mode: TimestamptzHandlingMode,
-    time_handling_mode: TimeHandlingMode,
-    custom_json_type: &CustomJsonType,
+    config: &JsonEncoderConfig,
 ) -> ArrayResult<Value> {
     let scalar_ref = match datum {
         None => {
@@ -223,7 +233,7 @@ fn datum_to_json_object(
             json!(v)
         }
         // Doris/Starrocks will convert out-of-bounds decimal and -INF, INF, NAN to NULL
-        (DataType::Decimal, ScalarRefImpl::Decimal(mut v)) => match custom_json_type {
+        (DataType::Decimal, ScalarRefImpl::Decimal(mut v)) => match &config.custom_json_type {
             CustomJsonType::Doris(map) => {
                 let s = map.get(&field.name).unwrap();
                 v.rescale(*s as u32);
@@ -233,21 +243,23 @@ fn datum_to_json_object(
                 json!(v.to_text())
             }
         },
-        (DataType::Timestamptz, ScalarRefImpl::Timestamptz(v)) => match timestamptz_handling_mode {
-            TimestamptzHandlingMode::UtcString => {
-                let parsed = v.to_datetime_utc();
-                let v = parsed.to_rfc3339_opts(chrono::SecondsFormat::Micros, true);
-                json!(v)
-            }
-            TimestamptzHandlingMode::UtcWithoutSuffix => {
-                let parsed = v.to_datetime_utc().naive_utc();
-                let v = parsed.format("%Y-%m-%d %H:%M:%S%.6f").to_string();
-                json!(v)
+        (DataType::Timestamptz, ScalarRefImpl::Timestamptz(v)) => {
+            match config.timestamptz_handling_mode {
+                TimestamptzHandlingMode::UtcString => {
+                    let parsed = v.to_datetime_utc();
+                    let v = parsed.to_rfc3339_opts(chrono::SecondsFormat::Micros, true);
+                    json!(v)
+                }
+                TimestamptzHandlingMode::UtcWithoutSuffix => {
+                    let parsed = v.to_datetime_utc().naive_utc();
+                    let v = parsed.format("%Y-%m-%d %H:%M:%S%.6f").to_string();
+                    json!(v)
+                }
+                TimestamptzHandlingMode::Micro => json!(v.timestamp_micros()),
+                TimestamptzHandlingMode::Milli => json!(v.timestamp_millis()),
             }
-            TimestamptzHandlingMode::Micro => json!(v.timestamp_micros()),
-            TimestamptzHandlingMode::Milli => json!(v.timestamp_millis()),
-        },
-        (DataType::Time, ScalarRefImpl::Time(v)) => match time_handling_mode {
+        }
+        (DataType::Time, ScalarRefImpl::Time(v)) => match config.time_handling_mode {
             TimeHandlingMode::Milli => {
                 // todo: just ignore the nanos part to avoid leap second complex
                 json!(v.0.num_seconds_from_midnight() as i64 * 1000)
@@ -257,7 +269,7 @@ fn datum_to_json_object(
                 json!(a)
             }
         },
-        (DataType::Date, ScalarRefImpl::Date(v)) => match date_handling_mode {
+        (DataType::Date, ScalarRefImpl::Date(v)) => match config.date_handling_mode {
             DateHandlingMode::FromCe => json!(v.0.num_days_from_ce()),
             DateHandlingMode::FromEpoch => {
                 let duration = v.0 - NaiveDateTime::UNIX_EPOCH.date();
@@ -268,10 +280,14 @@ fn datum_to_json_object(
                 json!(a)
             }
         },
-        (DataType::Timestamp, ScalarRefImpl::Timestamp(v)) => match timestamp_handling_mode {
-            TimestampHandlingMode::Milli => json!(v.0.and_utc().timestamp_millis()),
-            TimestampHandlingMode::String => json!(v.0.format("%Y-%m-%d %H:%M:%S%.6f").to_string()),
-        },
+        (DataType::Timestamp, ScalarRefImpl::Timestamp(v)) => {
+            match config.timestamp_handling_mode {
+                TimestampHandlingMode::Milli => json!(v.0.and_utc().timestamp_millis()),
+                TimestampHandlingMode::String => {
+                    json!(v.0.format("%Y-%m-%d %H:%M:%S%.6f").to_string())
+                }
+            }
+        }
         (DataType::Bytea, ScalarRefImpl::Bytea(v)) => {
             json!(general_purpose::STANDARD.encode(v))
         }
@@ -279,32 +295,25 @@ fn datum_to_json_object(
         (DataType::Interval, ScalarRefImpl::Interval(v)) => {
             json!(v.as_iso_8601())
         }
-        (DataType::Jsonb, ScalarRefImpl::Jsonb(jsonb_ref)) => match custom_json_type {
-            CustomJsonType::Es | CustomJsonType::StarRocks => JsonbVal::from(jsonb_ref).take(),
-            CustomJsonType::Doris(_) | CustomJsonType::None => {
+
+        (DataType::Jsonb, ScalarRefImpl::Jsonb(jsonb_ref)) => match config.jsonb_handling_mode {
+            JsonbHandlingMode::String => {
                 json!(jsonb_ref.to_string())
             }
+            JsonbHandlingMode::Dynamic => JsonbVal::from(jsonb_ref).take(),
         },
         (DataType::List(datatype), ScalarRefImpl::List(list_ref)) => {
             let elems = list_ref.iter();
             let mut vec = Vec::with_capacity(elems.len());
             let inner_field = Field::unnamed(Box::<DataType>::into_inner(datatype));
             for sub_datum_ref in elems {
-                let value = datum_to_json_object(
-                    &inner_field,
-                    sub_datum_ref,
-                    date_handling_mode,
-                    timestamp_handling_mode,
-                    timestamptz_handling_mode,
-                    time_handling_mode,
-                    custom_json_type,
-                )?;
+                let value = datum_to_json_object(&inner_field, sub_datum_ref, config)?;
                 vec.push(value);
             }
             json!(vec)
         }
         (DataType::Struct(st), ScalarRefImpl::Struct(struct_ref)) => {
-            match custom_json_type {
+            match config.custom_json_type {
                 CustomJsonType::Doris(_) => {
                     // We need to ensure that the order of elements in the json matches the insertion order.
                     let mut map = IndexMap::with_capacity(st.len());
@@ -312,15 +321,7 @@ fn datum_to_json_object(
                         st.iter()
                             .map(|(name, dt)| Field::with_name(dt.clone(), name)),
                     ) {
-                        let value = datum_to_json_object(
-                            &sub_field,
-                            sub_datum_ref,
-                            date_handling_mode,
-                            timestamp_handling_mode,
-                            timestamptz_handling_mode,
-                            time_handling_mode,
-                            custom_json_type,
-                        )?;
+                        let value = datum_to_json_object(&sub_field, sub_datum_ref, config)?;
                         map.insert(sub_field.name.clone(), value);
                     }
                     Value::String(
@@ -338,15 +339,7 @@ fn datum_to_json_object(
                         st.iter()
                             .map(|(name, dt)| Field::with_name(dt.clone(), name)),
                     ) {
-                        let value = datum_to_json_object(
-                            &sub_field,
-                            sub_datum_ref,
-                            date_handling_mode,
-                            timestamp_handling_mode,
-                            timestamptz_handling_mode,
-                            time_handling_mode,
-                            custom_json_type,
-                        )?;
+                        let value = datum_to_json_object(&sub_field, sub_datum_ref, config)?;
                         map.insert(sub_field.name.clone(), value);
                     }
                     json!(map)
@@ -454,17 +447,22 @@ mod tests {
             type_name: Default::default(),
         };
 
+        let config = JsonEncoderConfig {
+            time_handling_mode: TimeHandlingMode::Milli,
+            date_handling_mode: DateHandlingMode::FromCe,
+            timestamp_handling_mode: TimestampHandlingMode::String,
+            timestamptz_handling_mode: TimestamptzHandlingMode::UtcString,
+            custom_json_type: CustomJsonType::None,
+            jsonb_handling_mode: JsonbHandlingMode::String,
+        };
+
         let boolean_value = datum_to_json_object(
             &Field {
                 data_type: DataType::Boolean,
                 ..mock_field.clone()
             },
             Some(ScalarImpl::Bool(false).as_scalar_ref_impl()),
-            DateHandlingMode::FromCe,
-            TimestampHandlingMode::String,
-            TimestamptzHandlingMode::UtcString,
-            TimeHandlingMode::Milli,
-            &CustomJsonType::None,
+            &config,
         )
         .unwrap();
         assert_eq!(boolean_value, json!(false));
@@ -475,11 +473,7 @@ mod tests {
                 ..mock_field.clone()
             },
             Some(ScalarImpl::Int16(16).as_scalar_ref_impl()),
-            DateHandlingMode::FromCe,
-            TimestampHandlingMode::String,
-            TimestamptzHandlingMode::UtcString,
-            TimeHandlingMode::Milli,
-            &CustomJsonType::None,
+            &config,
         )
         .unwrap();
         assert_eq!(int16_value, json!(16));
@@ -490,11 +484,7 @@ mod tests {
                 ..mock_field.clone()
             },
             Some(ScalarImpl::Int64(i64::MAX).as_scalar_ref_impl()),
-            DateHandlingMode::FromCe,
-            TimestampHandlingMode::String,
-            TimestamptzHandlingMode::UtcString,
-            TimeHandlingMode::Milli,
-            &CustomJsonType::None,
+            &config,
         )
         .unwrap();
         assert_eq!(
@@ -508,11 +498,7 @@ mod tests {
                 ..mock_field.clone()
             },
             Some(ScalarImpl::Serial(i64::MAX.into()).as_scalar_ref_impl()),
-            DateHandlingMode::FromCe,
-            TimestampHandlingMode::String,
-            TimestamptzHandlingMode::UtcString,
-            TimeHandlingMode::Milli,
-            &CustomJsonType::None,
+            &config,
         )
         .unwrap();
         assert_eq!(
@@ -528,15 +514,20 @@ mod tests {
                 ..mock_field.clone()
             },
             Some(ScalarImpl::Timestamptz(tstz_inner).as_scalar_ref_impl()),
-            DateHandlingMode::FromCe,
-            TimestampHandlingMode::String,
-            TimestamptzHandlingMode::UtcString,
-            TimeHandlingMode::Milli,
-            &CustomJsonType::None,
+            &config,
         )
         .unwrap();
         assert_eq!(tstz_value, "2018-01-26T18:30:09.453000Z");
 
+        let unix_wo_suffix_config = JsonEncoderConfig {
+            time_handling_mode: TimeHandlingMode::Milli,
+            date_handling_mode: DateHandlingMode::FromCe,
+            timestamp_handling_mode: TimestampHandlingMode::String,
+            timestamptz_handling_mode: TimestamptzHandlingMode::UtcWithoutSuffix,
+            custom_json_type: CustomJsonType::None,
+            jsonb_handling_mode: JsonbHandlingMode::String,
+        };
+
         let tstz_inner = "2018-01-26T18:30:09.453Z".parse().unwrap();
         let tstz_value = datum_to_json_object(
             &Field {
@@ -544,15 +535,19 @@ mod tests {
                 ..mock_field.clone()
             },
             Some(ScalarImpl::Timestamptz(tstz_inner).as_scalar_ref_impl()),
-            DateHandlingMode::FromCe,
-            TimestampHandlingMode::String,
-            TimestamptzHandlingMode::UtcWithoutSuffix,
-            TimeHandlingMode::Milli,
-            &CustomJsonType::None,
+            &unix_wo_suffix_config,
         )
         .unwrap();
         assert_eq!(tstz_value, "2018-01-26 18:30:09.453000");
 
+        let timestamp_milli_config = JsonEncoderConfig {
+            time_handling_mode: TimeHandlingMode::String,
+            date_handling_mode: DateHandlingMode::FromCe,
+            timestamp_handling_mode: TimestampHandlingMode::Milli,
+            timestamptz_handling_mode: TimestamptzHandlingMode::UtcString,
+            custom_json_type: CustomJsonType::None,
+            jsonb_handling_mode: JsonbHandlingMode::String,
+        };
         let ts_value = datum_to_json_object(
             &Field {
                 data_type: DataType::Timestamp,
@@ -562,11 +557,7 @@ mod tests {
                 ScalarImpl::Timestamp(Timestamp::from_timestamp_uncheck(1000, 0))
                     .as_scalar_ref_impl(),
             ),
-            DateHandlingMode::FromCe,
-            TimestampHandlingMode::Milli,
-            TimestamptzHandlingMode::UtcString,
-            TimeHandlingMode::Milli,
-            &CustomJsonType::None,
+            &timestamp_milli_config,
         )
         .unwrap();
         assert_eq!(ts_value, json!(1000 * 1000));
@@ -580,11 +571,7 @@ mod tests {
                 ScalarImpl::Timestamp(Timestamp::from_timestamp_uncheck(1000, 0))
                     .as_scalar_ref_impl(),
             ),
-            DateHandlingMode::FromCe,
-            TimestampHandlingMode::String,
-            TimestamptzHandlingMode::UtcString,
-            TimeHandlingMode::Milli,
-            &CustomJsonType::None,
+            &config,
         )
         .unwrap();
         assert_eq!(ts_value, json!("1970-01-01 00:16:40.000000".to_string()));
@@ -599,11 +586,7 @@ mod tests {
                 ScalarImpl::Time(Time::from_num_seconds_from_midnight_uncheck(1000, 0))
                     .as_scalar_ref_impl(),
             ),
-            DateHandlingMode::FromCe,
-            TimestampHandlingMode::String,
-            TimestamptzHandlingMode::UtcString,
-            TimeHandlingMode::Milli,
-            &CustomJsonType::None,
+            &config,
         )
         .unwrap();
         assert_eq!(time_value, json!(1000 * 1000));
@@ -617,17 +600,21 @@ mod tests {
                 ScalarImpl::Interval(Interval::from_month_day_usec(13, 2, 1000000))
                     .as_scalar_ref_impl(),
             ),
-            DateHandlingMode::FromCe,
-            TimestampHandlingMode::String,
-            TimestamptzHandlingMode::UtcString,
-            TimeHandlingMode::Milli,
-            &CustomJsonType::None,
+            &config,
         )
         .unwrap();
         assert_eq!(interval_value, json!("P1Y1M2DT0H0M1S"));
 
         let mut map = HashMap::default();
         map.insert("aaa".to_string(), 5_u8);
+        let doris_config = JsonEncoderConfig {
+            time_handling_mode: TimeHandlingMode::String,
+            date_handling_mode: DateHandlingMode::String,
+            timestamp_handling_mode: TimestampHandlingMode::String,
+            timestamptz_handling_mode: TimestamptzHandlingMode::UtcString,
+            custom_json_type: CustomJsonType::Doris(map),
+            jsonb_handling_mode: JsonbHandlingMode::String,
+        };
         let decimal = datum_to_json_object(
             &Field {
                 data_type: DataType::Decimal,
@@ -635,11 +622,7 @@ mod tests {
                 ..mock_field.clone()
             },
             Some(ScalarImpl::Decimal(Decimal::try_from(1.1111111).unwrap()).as_scalar_ref_impl()),
-            DateHandlingMode::String,
-            TimestampHandlingMode::String,
-            TimestamptzHandlingMode::UtcString,
-            TimeHandlingMode::Milli,
-            &CustomJsonType::Doris(map),
+            &doris_config,
         )
         .unwrap();
         assert_eq!(decimal, json!("1.11111"));
@@ -650,41 +633,45 @@ mod tests {
                 ..mock_field.clone()
             },
             Some(ScalarImpl::Date(Date::from_ymd_uncheck(1970, 1, 1)).as_scalar_ref_impl()),
-            DateHandlingMode::FromCe,
-            TimestampHandlingMode::String,
-            TimestamptzHandlingMode::UtcString,
-            TimeHandlingMode::Milli,
-            &CustomJsonType::None,
+            &config,
         )
         .unwrap();
         assert_eq!(date_value, json!(719163));
 
+        let from_epoch_config = JsonEncoderConfig {
+            time_handling_mode: TimeHandlingMode::String,
+            date_handling_mode: DateHandlingMode::FromEpoch,
+            timestamp_handling_mode: TimestampHandlingMode::String,
+            timestamptz_handling_mode: TimestamptzHandlingMode::UtcString,
+            custom_json_type: CustomJsonType::None,
+            jsonb_handling_mode: JsonbHandlingMode::String,
+        };
         let date_value = datum_to_json_object(
             &Field {
                 data_type: DataType::Date,
                 ..mock_field.clone()
             },
             Some(ScalarImpl::Date(Date::from_ymd_uncheck(1970, 1, 1)).as_scalar_ref_impl()),
-            DateHandlingMode::FromEpoch,
-            TimestampHandlingMode::String,
-            TimestamptzHandlingMode::UtcString,
-            TimeHandlingMode::Milli,
-            &CustomJsonType::None,
+            &from_epoch_config,
         )
         .unwrap();
         assert_eq!(date_value, json!(0));
 
+        let doris_config = JsonEncoderConfig {
+            time_handling_mode: TimeHandlingMode::String,
+            date_handling_mode: DateHandlingMode::String,
+            timestamp_handling_mode: TimestampHandlingMode::String,
+            timestamptz_handling_mode: TimestamptzHandlingMode::UtcString,
+            custom_json_type: CustomJsonType::Doris(HashMap::default()),
+            jsonb_handling_mode: JsonbHandlingMode::String,
+        };
         let date_value = datum_to_json_object(
             &Field {
                 data_type: DataType::Date,
                 ..mock_field.clone()
             },
             Some(ScalarImpl::Date(Date::from_ymd_uncheck(2010, 10, 10)).as_scalar_ref_impl()),
-            DateHandlingMode::String,
-            TimestampHandlingMode::String,
-            TimestamptzHandlingMode::UtcString,
-            TimeHandlingMode::Milli,
-            &CustomJsonType::Doris(HashMap::default()),
+            &doris_config,
         )
         .unwrap();
         assert_eq!(date_value, json!("2010-10-10"));
@@ -705,14 +692,29 @@ mod tests {
                 ..mock_field.clone()
             },
             Some(ScalarRefImpl::Struct(StructRef::ValueRef { val: &value })),
-            DateHandlingMode::String,
-            TimestampHandlingMode::String,
-            TimestamptzHandlingMode::UtcString,
-            TimeHandlingMode::Milli,
-            &CustomJsonType::Doris(HashMap::default()),
+            &doris_config,
         )
         .unwrap();
         assert_eq!(interval_value, json!("{\"v3\":3,\"v2\":2,\"v1\":1}"));
+
+        let encode_jsonb_obj_config = JsonEncoderConfig {
+            time_handling_mode: TimeHandlingMode::String,
+            date_handling_mode: DateHandlingMode::String,
+            timestamp_handling_mode: TimestampHandlingMode::String,
+            timestamptz_handling_mode: TimestamptzHandlingMode::UtcString,
+            custom_json_type: CustomJsonType::None,
+            jsonb_handling_mode: JsonbHandlingMode::Dynamic,
+        };
+        let json_value = datum_to_json_object(
+            &Field {
+                data_type: DataType::Jsonb,
+                ..mock_field.clone()
+            },
+            Some(ScalarImpl::Jsonb(JsonbVal::from(json!([1, 2, 3]))).as_scalar_ref_impl()),
+            &encode_jsonb_obj_config,
+        )
+        .unwrap();
+        assert_eq!(json_value, json!([1, 2, 3]));
     }
 
     #[test]
diff --git a/src/connector/src/sink/encoder/mod.rs b/src/connector/src/sink/encoder/mod.rs
index 889d0162784bb..0a8a9e5abce73 100644
--- a/src/connector/src/sink/encoder/mod.rs
+++ b/src/connector/src/sink/encoder/mod.rs
@@ -150,6 +150,31 @@ pub enum CustomJsonType {
     None,
 }
 
+/// How the jsonb type is encoded.
+///
+/// - `String`: encode jsonb as string. `[1, true, "foo"] -> "[1, true, \"foo\"]"`
+/// - `Dynamic`: encode jsonb as json type dynamically. `[1, true, "foo"] -> [1, true, "foo"]`
+pub enum JsonbHandlingMode {
+    String,
+    Dynamic,
+}
+
+impl JsonbHandlingMode {
+    pub const OPTION_KEY: &'static str = "jsonb.handling.mode";
+
+    pub fn from_options(options: &BTreeMap<String, String>) -> Result<Self> {
+        match options.get(Self::OPTION_KEY).map(std::ops::Deref::deref) {
+            Some("string") | None => Ok(Self::String),
+            Some("dynamic") => Ok(Self::Dynamic),
+            Some(v) => Err(super::SinkError::Config(anyhow::anyhow!(
+                "unrecognized {} value {}",
+                Self::OPTION_KEY,
+                v
+            ))),
+        }
+    }
+}
+
 #[derive(Debug)]
 struct FieldEncodeError {
     message: String,
diff --git a/src/connector/src/sink/formatter/append_only.rs b/src/connector/src/sink/formatter/append_only.rs
index 8a4b7feda6a42..c0c47fa37e171 100644
--- a/src/connector/src/sink/formatter/append_only.rs
+++ b/src/connector/src/sink/formatter/append_only.rs
@@ -40,19 +40,22 @@ impl<KE: RowEncoder, VE: RowEncoder> SinkFormatter for AppendOnlyFormatter<KE, V
         &self,
         chunk: &StreamChunk,
     ) -> impl Iterator<Item = Result<(Option<Self::K>, Option<Self::V>)>> {
-        std::iter::from_coroutine(|| {
-            for (op, row) in chunk.rows() {
-                if op != Op::Insert {
-                    continue;
-                }
-                let event_key_object = match &self.key_encoder {
-                    Some(key_encoder) => Some(tri!(key_encoder.encode(row))),
-                    None => None,
-                };
-                let event_object = Some(tri!(self.val_encoder.encode(row)));
+        std::iter::from_coroutine(
+            #[coroutine]
+            || {
+                for (op, row) in chunk.rows() {
+                    if op != Op::Insert {
+                        continue;
+                    }
+                    let event_key_object = match &self.key_encoder {
+                        Some(key_encoder) => Some(tri!(key_encoder.encode(row))),
+                        None => None,
+                    };
+                    let event_object = Some(tri!(self.val_encoder.encode(row)));
 
-                yield Ok((event_key_object, event_object))
-            }
-        })
+                    yield Ok((event_key_object, event_object))
+                }
+            },
+        )
     }
 }
diff --git a/src/connector/src/sink/formatter/debezium_json.rs b/src/connector/src/sink/formatter/debezium_json.rs
index 6fff15058bd6f..a9bf0404f473e 100644
--- a/src/connector/src/sink/formatter/debezium_json.rs
+++ b/src/connector/src/sink/formatter/debezium_json.rs
@@ -21,8 +21,8 @@ use tracing::warn;
 
 use super::{Result, SinkFormatter, StreamChunk};
 use crate::sink::encoder::{
-    DateHandlingMode, JsonEncoder, RowEncoder, TimeHandlingMode, TimestampHandlingMode,
-    TimestamptzHandlingMode,
+    DateHandlingMode, JsonEncoder, JsonbHandlingMode, RowEncoder, TimeHandlingMode,
+    TimestampHandlingMode, TimestamptzHandlingMode,
 };
 use crate::tri;
 
@@ -69,6 +69,7 @@ impl DebeziumJsonFormatter {
             TimestampHandlingMode::Milli,
             TimestamptzHandlingMode::UtcString,
             TimeHandlingMode::Milli,
+            JsonbHandlingMode::String,
         );
         let val_encoder = JsonEncoder::new(
             schema.clone(),
@@ -77,6 +78,7 @@ impl DebeziumJsonFormatter {
             TimestampHandlingMode::Milli,
             TimestamptzHandlingMode::UtcString,
             TimeHandlingMode::Milli,
+            JsonbHandlingMode::String,
         );
         Self {
             schema,
@@ -98,100 +100,103 @@ impl SinkFormatter for DebeziumJsonFormatter {
         &self,
         chunk: &StreamChunk,
     ) -> impl Iterator<Item = Result<(Option<Value>, Option<Value>)>> {
-        std::iter::from_coroutine(|| {
-            let DebeziumJsonFormatter {
-                schema,
-                pk_indices,
-                db_name,
-                sink_from_name,
-                opts,
-                key_encoder,
-                val_encoder,
-            } = self;
-            let ts_ms = SystemTime::now()
-                .duration_since(UNIX_EPOCH)
-                .unwrap()
-                .as_millis() as u64;
-            let source_field = json!({
-                // todo: still some missing fields in source field
-                // ref https://debezium.io/documentation/reference/2.4/connectors/postgresql.html#postgresql-create-events
-                "db": db_name,
-                "table": sink_from_name,
-                "ts_ms": ts_ms,
-            });
-
-            let mut update_cache: Option<Map<String, Value>> = None;
-
-            for (op, row) in chunk.rows() {
-                let event_key_object: Option<Value> = Some(json!({
-                    "schema": json!({
-                        "type": "struct",
-                        "fields": fields_pk_to_json(&schema.fields, pk_indices),
-                        "optional": false,
-                        "name": concat_debezium_name_field(db_name, sink_from_name, "Key"),
-                    }),
-                    "payload": tri!(key_encoder.encode(row)),
-                }));
-                let event_object: Option<Value> = match op {
-                    Op::Insert => Some(json!({
-                        "schema": schema_to_json(schema, db_name, sink_from_name),
-                        "payload": {
-                            "before": null,
-                            "after": tri!(val_encoder.encode(row)),
-                            "op": "c",
-                            "ts_ms": ts_ms,
-                            "source": source_field,
-                        }
-                    })),
-                    Op::Delete => {
-                        let value_obj = Some(json!({
+        std::iter::from_coroutine(
+            #[coroutine]
+            || {
+                let DebeziumJsonFormatter {
+                    schema,
+                    pk_indices,
+                    db_name,
+                    sink_from_name,
+                    opts,
+                    key_encoder,
+                    val_encoder,
+                } = self;
+                let ts_ms = SystemTime::now()
+                    .duration_since(UNIX_EPOCH)
+                    .unwrap()
+                    .as_millis() as u64;
+                let source_field = json!({
+                    // todo: still some missing fields in source field
+                    // ref https://debezium.io/documentation/reference/2.4/connectors/postgresql.html#postgresql-create-events
+                    "db": db_name,
+                    "table": sink_from_name,
+                    "ts_ms": ts_ms,
+                });
+
+                let mut update_cache: Option<Map<String, Value>> = None;
+
+                for (op, row) in chunk.rows() {
+                    let event_key_object: Option<Value> = Some(json!({
+                        "schema": json!({
+                            "type": "struct",
+                            "fields": fields_pk_to_json(&schema.fields, pk_indices),
+                            "optional": false,
+                            "name": concat_debezium_name_field(db_name, sink_from_name, "Key"),
+                        }),
+                        "payload": tri!(key_encoder.encode(row)),
+                    }));
+                    let event_object: Option<Value> = match op {
+                        Op::Insert => Some(json!({
                             "schema": schema_to_json(schema, db_name, sink_from_name),
                             "payload": {
-                                "before": tri!(val_encoder.encode(row)),
-                                "after": null,
-                                "op": "d",
+                                "before": null,
+                                "after": tri!(val_encoder.encode(row)),
+                                "op": "c",
                                 "ts_ms": ts_ms,
                                 "source": source_field,
                             }
-                        }));
-                        yield Ok((event_key_object.clone(), value_obj));
-
-                        if opts.gen_tombstone {
-                            // Tomestone event
-                            // https://debezium.io/documentation/reference/2.1/connectors/postgresql.html#postgresql-delete-events
-                            yield Ok((event_key_object, None));
-                        }
-
-                        continue;
-                    }
-                    Op::UpdateDelete => {
-                        update_cache = Some(tri!(val_encoder.encode(row)));
-                        continue;
-                    }
-                    Op::UpdateInsert => {
-                        if let Some(before) = update_cache.take() {
-                            Some(json!({
+                        })),
+                        Op::Delete => {
+                            let value_obj = Some(json!({
                                 "schema": schema_to_json(schema, db_name, sink_from_name),
                                 "payload": {
-                                    "before": before,
-                                    "after": tri!(val_encoder.encode(row)),
-                                    "op": "u",
+                                    "before": tri!(val_encoder.encode(row)),
+                                    "after": null,
+                                    "op": "d",
                                     "ts_ms": ts_ms,
                                     "source": source_field,
                                 }
-                            }))
-                        } else {
-                            warn!(
-                                "not found UpdateDelete in prev row, skipping, row index {:?}",
-                                row.index()
-                            );
+                            }));
+                            yield Ok((event_key_object.clone(), value_obj));
+
+                            if opts.gen_tombstone {
+                                // Tomestone event
+                                // https://debezium.io/documentation/reference/2.1/connectors/postgresql.html#postgresql-delete-events
+                                yield Ok((event_key_object, None));
+                            }
+
                             continue;
                         }
-                    }
-                };
-                yield Ok((event_key_object, event_object));
-            }
-        })
+                        Op::UpdateDelete => {
+                            update_cache = Some(tri!(val_encoder.encode(row)));
+                            continue;
+                        }
+                        Op::UpdateInsert => {
+                            if let Some(before) = update_cache.take() {
+                                Some(json!({
+                                    "schema": schema_to_json(schema, db_name, sink_from_name),
+                                    "payload": {
+                                        "before": before,
+                                        "after": tri!(val_encoder.encode(row)),
+                                        "op": "u",
+                                        "ts_ms": ts_ms,
+                                        "source": source_field,
+                                    }
+                                }))
+                            } else {
+                                warn!(
+                                    "not found UpdateDelete in prev row, skipping, row index {:?}",
+                                    row.index()
+                                );
+                                continue;
+                            }
+                        }
+                    };
+                    yield Ok((event_key_object, event_object));
+                }
+            },
+        )
     }
 }
 
@@ -397,6 +402,7 @@ mod tests {
             TimestampHandlingMode::Milli,
             TimestamptzHandlingMode::UtcString,
             TimeHandlingMode::Milli,
+            JsonbHandlingMode::String,
         );
         let json_chunk = chunk_to_json(chunk, &encoder).unwrap();
         let schema_json = schema_to_json(&schema, "test_db", "test_table");
diff --git a/src/connector/src/sink/formatter/mod.rs b/src/connector/src/sink/formatter/mod.rs
index 4628a925da98d..b2e93cba763ea 100644
--- a/src/connector/src/sink/formatter/mod.rs
+++ b/src/connector/src/sink/formatter/mod.rs
@@ -31,7 +31,8 @@ use super::catalog::{SinkEncode, SinkFormat, SinkFormatDesc};
 use super::encoder::template::TemplateEncoder;
 use super::encoder::text::TextEncoder;
 use super::encoder::{
-    DateHandlingMode, KafkaConnectParams, TimeHandlingMode, TimestamptzHandlingMode,
+    DateHandlingMode, JsonbHandlingMode, KafkaConnectParams, TimeHandlingMode,
+    TimestamptzHandlingMode,
 };
 use super::redis::{KEY_FORMAT, VALUE_FORMAT};
 use crate::sink::encoder::{
@@ -113,6 +114,7 @@ pub trait EncoderBuild: Sized {
 impl EncoderBuild for JsonEncoder {
     async fn build(b: EncoderParams<'_>, pk_indices: Option<Vec<usize>>) -> Result<Self> {
         let timestamptz_mode = TimestamptzHandlingMode::from_options(&b.format_desc.options)?;
+        let jsonb_handling_mode = JsonbHandlingMode::from_options(&b.format_desc.options)?;
         let encoder = JsonEncoder::new(
             b.schema,
             pk_indices,
@@ -120,6 +122,7 @@ impl EncoderBuild for JsonEncoder {
             TimestampHandlingMode::Milli,
             timestamptz_mode,
             TimeHandlingMode::Milli,
+            jsonb_handling_mode,
         );
         let encoder = if let Some(s) = b.format_desc.options.get("schemas.enable") {
             match s.to_lowercase().parse::<bool>() {
diff --git a/src/connector/src/sink/formatter/upsert.rs b/src/connector/src/sink/formatter/upsert.rs
index 612c22aaaf86c..7e586a7917ea1 100644
--- a/src/connector/src/sink/formatter/upsert.rs
+++ b/src/connector/src/sink/formatter/upsert.rs
@@ -40,22 +40,25 @@ impl<KE: RowEncoder, VE: RowEncoder> SinkFormatter for UpsertFormatter<KE, VE> {
         &self,
         chunk: &StreamChunk,
     ) -> impl Iterator<Item = Result<(Option<Self::K>, Option<Self::V>)>> {
-        std::iter::from_coroutine(|| {
-            for (op, row) in chunk.rows() {
-                let event_key_object = Some(tri!(self.key_encoder.encode(row)));
-
-                let event_object = match op {
-                    Op::Insert | Op::UpdateInsert => Some(tri!(self.val_encoder.encode(row))),
-                    // Empty value with a key
-                    Op::Delete => None,
-                    Op::UpdateDelete => {
-                        // upsert semantic does not require update delete event
-                        continue;
-                    }
-                };
-
-                yield Ok((event_key_object, event_object))
-            }
-        })
+        std::iter::from_coroutine(
+            #[coroutine]
+            || {
+                for (op, row) in chunk.rows() {
+                    let event_key_object = Some(tri!(self.key_encoder.encode(row)));
+
+                    let event_object = match op {
+                        Op::Insert | Op::UpdateInsert => Some(tri!(self.val_encoder.encode(row))),
+                        // Empty value with a key
+                        Op::Delete => None,
+                        Op::UpdateDelete => {
+                            // upsert semantic does not require update delete event
+                            continue;
+                        }
+                    };
+
+                    yield Ok((event_key_object, event_object))
+                }
+            },
+        )
     }
 }
diff --git a/src/connector/src/sink/iceberg/mod.rs b/src/connector/src/sink/iceberg/mod.rs
index 4399b9c45334e..a17c98985de0c 100644
--- a/src/connector/src/sink/iceberg/mod.rs
+++ b/src/connector/src/sink/iceberg/mod.rs
@@ -29,6 +29,7 @@ use arrow_schema_iceberg::{
 };
 use async_trait::async_trait;
 use iceberg::io::{S3_ACCESS_KEY_ID, S3_ENDPOINT, S3_REGION, S3_SECRET_ACCESS_KEY};
+use iceberg::spec::TableMetadata;
 use iceberg::table::Table as TableV2;
 use iceberg::{Catalog as CatalogV2, TableIdent};
 use icelake::catalog::{
@@ -47,7 +48,7 @@ use itertools::Itertools;
 use risingwave_common::array::arrow::IcebergArrowConvert;
 use risingwave_common::array::{Op, StreamChunk};
 use risingwave_common::bail;
-use risingwave_common::buffer::Bitmap;
+use risingwave_common::bitmap::Bitmap;
 use risingwave_common::catalog::Schema;
 use risingwave_pb::connector_service::sink_metadata::Metadata::Serialized;
 use risingwave_pb::connector_service::sink_metadata::SerializedMetadata;
@@ -514,10 +515,26 @@ impl IcebergConfig {
                 Ok(Arc::new(catalog))
             }
             "rest" => {
+                let mut iceberg_configs = HashMap::new();
+                if let Some(region) = &self.region {
+                    iceberg_configs.insert(S3_REGION.to_string(), region.clone().to_string());
+                }
+                if let Some(endpoint) = &self.endpoint {
+                    iceberg_configs.insert(S3_ENDPOINT.to_string(), endpoint.clone().to_string());
+                }
+                iceberg_configs.insert(
+                    S3_ACCESS_KEY_ID.to_string(),
+                    self.access_key.clone().to_string(),
+                );
+                iceberg_configs.insert(
+                    S3_SECRET_ACCESS_KEY.to_string(),
+                    self.secret_key.clone().to_string(),
+                );
                 let config = iceberg_catalog_rest::RestCatalogConfig::builder()
                     .uri(self.uri.clone().ok_or_else(|| {
                         SinkError::Iceberg(anyhow!("`catalog.uri` must be set in rest catalog"))
                     })?)
+                    .props(iceberg_configs)
                     .build();
                 let catalog = iceberg_catalog_rest::RestCatalog::new(config).await?;
                 Ok(Arc::new(catalog))
@@ -562,6 +579,37 @@ impl IcebergConfig {
 
         catalog.load_table(&table_id).await.map_err(Into::into)
     }
+
+    pub async fn load_table_v2_with_metadata(
+        &self,
+        metadata: TableMetadata,
+    ) -> ConnectorResult<TableV2> {
+        match self.catalog_type() {
+            "storage" => {
+                let config = StorageCatalogConfig::builder()
+                    .warehouse(self.path.clone())
+                    .access_key(self.access_key.clone())
+                    .secret_key(self.secret_key.clone())
+                    .region(self.region.clone())
+                    .endpoint(self.endpoint.clone())
+                    .build();
+                let storage_catalog = storage_catalog::StorageCatalog::new(config)?;
+
+                let table_id = self
+                    .full_table_name_v2()
+                    .context("Unable to parse table name")?;
+
+                Ok(iceberg::table::Table::builder()
+                    .metadata(metadata)
+                    .identifier(table_id)
+                    .file_io(storage_catalog.file_io().clone())
+                    // Only support readonly table for storage catalog now.
+                    .readonly(true)
+                    .build())
+            }
+            _ => self.load_table_v2().await,
+        }
+    }
 }
 
 pub struct IcebergSink {
@@ -696,7 +744,7 @@ impl Sink for IcebergSink {
             self.param.clone(),
             writer_param.vnode_bitmap.ok_or_else(|| {
                 SinkError::Remote(anyhow!(
-                    "sink needs coordination should not have singleton input"
+                    "sink needs coordination and should not have singleton input"
                 ))
             })?,
             inner,
@@ -716,10 +764,12 @@ impl Sink for IcebergSink {
     }
 
     async fn new_coordinator(&self) -> Result<Self::Coordinator> {
+        let catalog = self.config.create_catalog().await?;
         let table = self.create_and_validate_table().await?;
         let partition_type = table.current_partition_type()?;
 
         Ok(IcebergSinkCommitter {
+            catalog,
             table,
             partition_type,
         })
@@ -1012,7 +1062,7 @@ impl WriteResult {
             {
                 v
             } else {
-                bail!("iceberg sink metadata should be a object");
+                bail!("iceberg sink metadata should be an object");
             };
 
             let data_files: Vec<DataFile>;
@@ -1039,7 +1089,7 @@ impl WriteResult {
                     .collect::<std::result::Result<Vec<DataFile>, icelake::Error>>()
                     .context("Failed to parse data file from json")?;
             } else {
-                bail!("icberg sink metadata should have data_files object");
+                bail!("Iceberg sink metadata should have data_files object");
             }
             Ok(Self {
                 data_files,
@@ -1091,6 +1141,7 @@ impl<'a> TryFrom<&'a WriteResult> for SinkMetadata {
 }
 
 pub struct IcebergSinkCommitter {
+    catalog: CatalogRef,
     table: Table,
     partition_type: Any,
 }
@@ -1117,6 +1168,12 @@ impl SinkCommitCoordinator for IcebergSinkCommitter {
             tracing::debug!(?epoch, "no data to commit");
             return Ok(());
         }
+        // Load the latest table to avoid concurrent modification with the best effort.
+        self.table = self
+            .catalog
+            .clone()
+            .load_table(self.table.table_name())
+            .await?;
         let mut txn = Transaction::new(&mut self.table);
         write_results.into_iter().for_each(|s| {
             txn.append_data_file(s.data_files);
@@ -1139,7 +1196,7 @@ pub fn try_matches_arrow_schema(
 ) -> anyhow::Result<()> {
     if rw_schema.fields.len() != arrow_schema.fields().len() {
         bail!(
-            "Schema length not match, risingwave is {}, and iceberg is {}",
+            "Schema length mismatch, risingwave is {}, and iceberg is {}",
             rw_schema.fields.len(),
             arrow_schema.fields.len()
         );
diff --git a/src/connector/src/sink/iceberg/prometheus/monitored_base_file_writer.rs b/src/connector/src/sink/iceberg/prometheus/monitored_base_file_writer.rs
index 3c205fd3b104e..c29608f0fd63b 100644
--- a/src/connector/src/sink/iceberg/prometheus/monitored_base_file_writer.rs
+++ b/src/connector/src/sink/iceberg/prometheus/monitored_base_file_writer.rs
@@ -25,13 +25,13 @@ use risingwave_common::metrics::LabelGuardedIntGauge;
 pub struct MonitoredBaseFileWriterBuilder<B: FileWriterBuilder> {
     inner: BaseFileWriterBuilder<B>,
     // metrics
-    unflush_data_file: LabelGuardedIntGauge<2>,
+    unflush_data_file: LabelGuardedIntGauge<3>,
 }
 
 impl<B: FileWriterBuilder> MonitoredBaseFileWriterBuilder<B> {
     pub fn new(
         inner: BaseFileWriterBuilder<B>,
-        unflush_data_file: LabelGuardedIntGauge<2>,
+        unflush_data_file: LabelGuardedIntGauge<3>,
     ) -> Self {
         Self {
             inner,
@@ -59,7 +59,7 @@ pub struct MonitoredBaseFileWriter<B: FileWriterBuilder> {
     inner: BaseFileWriter<B>,
 
     // metrics
-    unflush_data_file: LabelGuardedIntGauge<2>,
+    unflush_data_file: LabelGuardedIntGauge<3>,
 
     cur_metrics: BaseFileWriterMetrics,
 }
diff --git a/src/connector/src/sink/iceberg/prometheus/monitored_position_delete_writer.rs b/src/connector/src/sink/iceberg/prometheus/monitored_position_delete_writer.rs
index 43314c3fae384..8be6eb7018b13 100644
--- a/src/connector/src/sink/iceberg/prometheus/monitored_position_delete_writer.rs
+++ b/src/connector/src/sink/iceberg/prometheus/monitored_position_delete_writer.rs
@@ -21,14 +21,14 @@ use risingwave_common::metrics::LabelGuardedIntGauge;
 
 #[derive(Clone)]
 pub struct MonitoredPositionDeleteWriterBuilder<B: FileWriterBuilder> {
-    current_cache_number: LabelGuardedIntGauge<2>,
+    current_cache_number: LabelGuardedIntGauge<3>,
     inner: PositionDeleteWriterBuilder<B>,
 }
 
 impl<B: FileWriterBuilder> MonitoredPositionDeleteWriterBuilder<B> {
     pub fn new(
         inner: PositionDeleteWriterBuilder<B>,
-        current_cache_number: LabelGuardedIntGauge<2>,
+        current_cache_number: LabelGuardedIntGauge<3>,
     ) -> Self {
         Self {
             current_cache_number,
@@ -59,7 +59,7 @@ pub struct MonitoredPositionDeleteWriter<B: FileWriterBuilder> {
     writer: PositionDeleteWriter<B>,
 
     // metrics
-    cache_number: LabelGuardedIntGauge<2>,
+    cache_number: LabelGuardedIntGauge<3>,
     current_metrics: PositionDeleteMetrics,
 }
 
diff --git a/src/connector/src/sink/iceberg/storage_catalog.rs b/src/connector/src/sink/iceberg/storage_catalog.rs
index 5fb2c5105fda5..7fd025020e1d9 100644
--- a/src/connector/src/sink/iceberg/storage_catalog.rs
+++ b/src/connector/src/sink/iceberg/storage_catalog.rs
@@ -162,6 +162,10 @@ impl StorageCatalog {
 
         Ok(paths)
     }
+
+    pub fn file_io(&self) -> &FileIO {
+        &self.file_io
+    }
 }
 
 #[async_trait]
diff --git a/src/connector/src/sink/kafka.rs b/src/connector/src/sink/kafka.rs
index 617f427ae71f1..3a6914d2249c5 100644
--- a/src/connector/src/sink/kafka.rs
+++ b/src/connector/src/sink/kafka.rs
@@ -403,7 +403,16 @@ struct KafkaPayloadWriter<'a> {
     config: &'a KafkaConfig,
 }
 
-pub type KafkaSinkDeliveryFuture = impl TryFuture<Ok = (), Error = SinkError> + Unpin + 'static;
+mod opaque_type {
+    use super::*;
+    pub type KafkaSinkDeliveryFuture = impl TryFuture<Ok = (), Error = SinkError> + Unpin + 'static;
+
+    pub(super) fn map_delivery_future(future: DeliveryFuture) -> KafkaSinkDeliveryFuture {
+        future.map(KafkaPayloadWriter::<'static>::map_future_result)
+    }
+}
+use opaque_type::map_delivery_future;
+pub use opaque_type::KafkaSinkDeliveryFuture;
 
 pub struct KafkaSinkWriter {
     formatter: SinkFormatterImpl,
@@ -482,7 +491,7 @@ impl<'w> KafkaPayloadWriter<'w> {
                 Ok(delivery_future) => {
                     if self
                         .add_future
-                        .add_future_may_await(Self::map_delivery_future(delivery_future))
+                        .add_future_may_await(map_delivery_future(delivery_future))
                         .await?
                     {
                         tracing::warn!(
@@ -567,10 +576,6 @@ impl<'w> KafkaPayloadWriter<'w> {
             Err(_) => Err(KafkaError::Canceled.into()),
         }
     }
-
-    fn map_delivery_future(future: DeliveryFuture) -> KafkaSinkDeliveryFuture {
-        future.map(KafkaPayloadWriter::<'static>::map_future_result)
-    }
 }
 
 impl<'a> FormattedSink for KafkaPayloadWriter<'a> {
@@ -590,7 +595,7 @@ mod test {
 
     use super::*;
     use crate::sink::encoder::{
-        DateHandlingMode, JsonEncoder, TimeHandlingMode, TimestampHandlingMode,
+        DateHandlingMode, JsonEncoder, JsonbHandlingMode, TimeHandlingMode, TimestampHandlingMode,
         TimestamptzHandlingMode,
     };
     use crate::sink::formatter::AppendOnlyFormatter;
@@ -778,6 +783,7 @@ mod test {
                     TimestampHandlingMode::Milli,
                     TimestamptzHandlingMode::UtcString,
                     TimeHandlingMode::Milli,
+                    JsonbHandlingMode::String,
                 ),
             )),
         )
diff --git a/src/connector/src/sink/kinesis.rs b/src/connector/src/sink/kinesis.rs
index 771d3c8a6f91d..3b5d49da46037 100644
--- a/src/connector/src/sink/kinesis.rs
+++ b/src/connector/src/sink/kinesis.rs
@@ -201,8 +201,36 @@ impl KinesisSinkWriter {
     }
 }
 
-pub type KinesisSinkPayloadWriterDeliveryFuture =
-    impl TryFuture<Ok = (), Error = SinkError> + Unpin + Send + 'static;
+mod opaque_type {
+    use super::*;
+    pub type KinesisSinkPayloadWriterDeliveryFuture =
+        impl TryFuture<Ok = (), Error = SinkError> + Unpin + Send + 'static;
+
+    impl KinesisSinkPayloadWriter {
+        pub(super) fn finish(self) -> KinesisSinkPayloadWriterDeliveryFuture {
+            async move {
+                let builder = self.builder.expect("should not be None");
+                let context_fmt = format!(
+                    "failed to put record to {}",
+                    builder
+                        .get_stream_name()
+                        .as_ref()
+                        .expect("should have set stream name")
+                );
+                Retry::spawn(
+                    ExponentialBackoff::from_millis(100).map(jitter).take(3),
+                    || builder.clone().send(),
+                )
+                .await
+                .with_context(|| context_fmt.clone())
+                .map_err(SinkError::Kinesis)?;
+                Ok(())
+            }
+            .boxed()
+        }
+    }
+}
+pub use opaque_type::KinesisSinkPayloadWriterDeliveryFuture;
 
 impl KinesisSinkPayloadWriter {
     fn put_record(&mut self, key: String, payload: Vec<u8>) {
@@ -216,28 +244,6 @@ impl KinesisSinkPayloadWriter {
             ),
         );
     }
-
-    fn finish(self) -> KinesisSinkPayloadWriterDeliveryFuture {
-        async move {
-            let builder = self.builder.expect("should not be None");
-            let context_fmt = format!(
-                "failed to put record to {}",
-                builder
-                    .get_stream_name()
-                    .as_ref()
-                    .expect("should have set stream name")
-            );
-            Retry::spawn(
-                ExponentialBackoff::from_millis(100).map(jitter).take(3),
-                || builder.clone().send(),
-            )
-            .await
-            .with_context(|| context_fmt.clone())
-            .map_err(SinkError::Kinesis)?;
-            Ok(())
-        }
-        .boxed()
-    }
 }
 
 impl FormattedSink for KinesisSinkPayloadWriter {
diff --git a/src/connector/src/sink/log_store.rs b/src/connector/src/sink/log_store.rs
index d609aa7170d0d..3c25b1ec7d75a 100644
--- a/src/connector/src/sink/log_store.rs
+++ b/src/connector/src/sink/log_store.rs
@@ -24,7 +24,7 @@ use await_tree::InstrumentAwait;
 use futures::{TryFuture, TryFutureExt};
 use risingwave_common::array::StreamChunk;
 use risingwave_common::bail;
-use risingwave_common::buffer::Bitmap;
+use risingwave_common::bitmap::Bitmap;
 use risingwave_common::metrics::LabelGuardedIntCounter;
 use risingwave_common::util::epoch::{EpochPair, INVALID_EPOCH};
 
@@ -85,7 +85,7 @@ impl TruncateOffset {
             } => {
                 if epoch != *offset_epoch {
                     bail!(
-                        "new item epoch {} not match current chunk offset epoch {}",
+                        "new item epoch {} does not match current chunk offset epoch {}",
                         epoch,
                         offset_epoch
                     );
@@ -96,7 +96,7 @@ impl TruncateOffset {
             } => {
                 if epoch <= *offset_epoch {
                     bail!(
-                        "new item epoch {} not exceed barrier offset epoch {}",
+                        "new item epoch {} does not exceed barrier offset epoch {}",
                         epoch,
                         offset_epoch
                     );
@@ -220,11 +220,11 @@ pub struct BackpressureMonitoredLogReader<R: LogReader> {
     inner: R,
     /// Start time to wait for new future after poll ready
     wait_new_future_start_time: Option<Instant>,
-    wait_new_future_duration_ns: LabelGuardedIntCounter<3>,
+    wait_new_future_duration_ns: LabelGuardedIntCounter<4>,
 }
 
 impl<R: LogReader> BackpressureMonitoredLogReader<R> {
-    fn new(inner: R, wait_new_future_duration_ns: LabelGuardedIntCounter<3>) -> Self {
+    fn new(inner: R, wait_new_future_duration_ns: LabelGuardedIntCounter<4>) -> Self {
         Self {
             inner,
             wait_new_future_start_time: None,
diff --git a/src/connector/src/sink/mock_coordination_client.rs b/src/connector/src/sink/mock_coordination_client.rs
index 8089a99e32ec0..61a1a163171b2 100644
--- a/src/connector/src/sink/mock_coordination_client.rs
+++ b/src/connector/src/sink/mock_coordination_client.rs
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use risingwave_common::buffer::Bitmap;
+use risingwave_common::bitmap::Bitmap;
 use risingwave_pb::connector_service::coordinate_response::{
     self, CommitResponse, StartCoordinationResponse,
 };
diff --git a/src/connector/src/sink/mod.rs b/src/connector/src/sink/mod.rs
index 097c0e57b15c1..cad35a278edb0 100644
--- a/src/connector/src/sink/mod.rs
+++ b/src/connector/src/sink/mod.rs
@@ -38,7 +38,6 @@ pub mod pulsar;
 pub mod redis;
 pub mod remote;
 pub mod snowflake;
-pub mod snowflake_connector;
 pub mod sqlserver;
 pub mod starrocks;
 pub mod test_sink;
@@ -54,11 +53,12 @@ use ::deltalake::DeltaTableError;
 use ::redis::RedisError;
 use anyhow::anyhow;
 use async_trait::async_trait;
-use risingwave_common::buffer::Bitmap;
+use risingwave_common::bitmap::Bitmap;
 use risingwave_common::catalog::{ColumnDesc, Field, Schema};
 use risingwave_common::metrics::{
     LabelGuardedHistogram, LabelGuardedIntCounter, LabelGuardedIntGauge,
 };
+use risingwave_common::secret::{LocalSecretManager, SecretError};
 use risingwave_common::session_config::sink_decouple::SinkDecouple;
 use risingwave_pb::catalog::PbSinkType;
 use risingwave_pb::connector_service::{PbSinkParam, SinkMetadata, TableSchema};
@@ -231,45 +231,61 @@ impl SinkParam {
             fields: self.columns.iter().map(Field::from).collect(),
         }
     }
-}
 
-impl From<SinkCatalog> for SinkParam {
-    fn from(sink_catalog: SinkCatalog) -> Self {
+    // `SinkParams` should only be used when there is a secret context.
+    // FIXME: Use a new type for `SinkFormatDesc` with properties contain filled secrets.
+    pub fn fill_secret_for_format_desc(
+        format_desc: Option<SinkFormatDesc>,
+    ) -> Result<Option<SinkFormatDesc>> {
+        match format_desc {
+            Some(mut format_desc) => {
+                format_desc.options = LocalSecretManager::global()
+                    .fill_secrets(format_desc.options, format_desc.secret_refs.clone())?;
+                Ok(Some(format_desc))
+            }
+            None => Ok(None),
+        }
+    }
+
+    /// Try to convert a `SinkCatalog` to a `SinkParam` and fill the secrets to properties.
+    pub fn try_from_sink_catalog(sink_catalog: SinkCatalog) -> Result<Self> {
         let columns = sink_catalog
             .visible_columns()
             .map(|col| col.column_desc.clone())
             .collect();
-        Self {
+        let properties_with_secret = LocalSecretManager::global()
+            .fill_secrets(sink_catalog.properties, sink_catalog.secret_refs)?;
+        let format_desc_with_secret = Self::fill_secret_for_format_desc(sink_catalog.format_desc)?;
+        Ok(Self {
             sink_id: sink_catalog.id,
             sink_name: sink_catalog.name,
-            properties: sink_catalog.properties,
+            properties: properties_with_secret,
             columns,
             downstream_pk: sink_catalog.downstream_pk,
             sink_type: sink_catalog.sink_type,
-            format_desc: sink_catalog.format_desc,
+            format_desc: format_desc_with_secret,
             db_name: sink_catalog.db_name,
             sink_from_name: sink_catalog.sink_from_name,
-        }
+        })
     }
 }
 
 #[derive(Clone)]
 pub struct SinkMetrics {
-    pub sink_commit_duration_metrics: LabelGuardedHistogram<3>,
-    pub connector_sink_rows_received: LabelGuardedIntCounter<2>,
-    pub log_store_first_write_epoch: LabelGuardedIntGauge<3>,
-    pub log_store_latest_write_epoch: LabelGuardedIntGauge<3>,
-    pub log_store_write_rows: LabelGuardedIntCounter<3>,
-    pub log_store_latest_read_epoch: LabelGuardedIntGauge<3>,
-    pub log_store_read_rows: LabelGuardedIntCounter<3>,
-
-    pub log_store_reader_wait_new_future_duration_ns: LabelGuardedIntCounter<3>,
-
-    pub iceberg_write_qps: LabelGuardedIntCounter<2>,
-    pub iceberg_write_latency: LabelGuardedHistogram<2>,
-    pub iceberg_rolling_unflushed_data_file: LabelGuardedIntGauge<2>,
-    pub iceberg_position_delete_cache_num: LabelGuardedIntGauge<2>,
-    pub iceberg_partition_num: LabelGuardedIntGauge<2>,
+    pub sink_commit_duration_metrics: LabelGuardedHistogram<4>,
+    pub connector_sink_rows_received: LabelGuardedIntCounter<3>,
+    pub log_store_first_write_epoch: LabelGuardedIntGauge<4>,
+    pub log_store_latest_write_epoch: LabelGuardedIntGauge<4>,
+    pub log_store_write_rows: LabelGuardedIntCounter<4>,
+    pub log_store_latest_read_epoch: LabelGuardedIntGauge<4>,
+    pub log_store_read_rows: LabelGuardedIntCounter<4>,
+    pub log_store_reader_wait_new_future_duration_ns: LabelGuardedIntCounter<4>,
+
+    pub iceberg_write_qps: LabelGuardedIntCounter<3>,
+    pub iceberg_write_latency: LabelGuardedHistogram<3>,
+    pub iceberg_rolling_unflushed_data_file: LabelGuardedIntGauge<3>,
+    pub iceberg_position_delete_cache_num: LabelGuardedIntGauge<3>,
+    pub iceberg_partition_num: LabelGuardedIntGauge<3>,
 }
 
 impl SinkMetrics {
@@ -598,6 +614,12 @@ pub enum SinkError {
         #[backtrace]
         ConnectorError,
     ),
+    #[error("Secret error: {0}")]
+    Secret(
+        #[from]
+        #[backtrace]
+        SecretError,
+    ),
     #[error("Mongodb error: {0}")]
     Mongodb(
         #[source]
diff --git a/src/connector/src/sink/mqtt.rs b/src/connector/src/sink/mqtt.rs
index 9e86b5ec97e2a..896da3a7f4d89 100644
--- a/src/connector/src/sink/mqtt.rs
+++ b/src/connector/src/sink/mqtt.rs
@@ -31,8 +31,8 @@ use with_options::WithOptions;
 
 use super::catalog::{SinkEncode, SinkFormat, SinkFormatDesc};
 use super::encoder::{
-    DateHandlingMode, JsonEncoder, ProtoEncoder, ProtoHeader, RowEncoder, SerTo, TimeHandlingMode,
-    TimestampHandlingMode, TimestamptzHandlingMode,
+    DateHandlingMode, JsonEncoder, JsonbHandlingMode, ProtoEncoder, ProtoHeader, RowEncoder, SerTo,
+    TimeHandlingMode, TimestampHandlingMode, TimestamptzHandlingMode,
 };
 use super::writer::AsyncTruncateSinkWriterExt;
 use super::{DummySinkCommitCoordinator, SinkWriterParam};
@@ -133,7 +133,7 @@ impl MqttConfig {
             .map_err(|e| SinkError::Config(anyhow!(e)))?;
         if config.r#type != SINK_TYPE_APPEND_ONLY {
             Err(SinkError::Config(anyhow!(
-                "Mqtt sink only support append-only mode"
+                "MQTT sink only supports append-only mode"
             )))
         } else {
             Ok(config)
@@ -175,7 +175,7 @@ impl Sink for MqttSink {
     async fn validate(&self) -> Result<()> {
         if !self.is_append_only {
             return Err(SinkError::Mqtt(anyhow!(
-                "Mqtt sink only support append-only mode"
+                "MQTT sink only supports append-only mode"
             )));
         }
 
@@ -221,7 +221,7 @@ impl MqttSinkWriter {
         }
 
         let timestamptz_mode = TimestamptzHandlingMode::from_options(&format_desc.options)?;
-
+        let jsonb_handling_mode = JsonbHandlingMode::from_options(&format_desc.options)?;
         let encoder = match format_desc.format {
             SinkFormat::AppendOnly => match format_desc.encode {
                 SinkEncode::Json => RowEncoderWrapper::Json(JsonEncoder::new(
@@ -231,6 +231,7 @@ impl MqttSinkWriter {
                     TimestampHandlingMode::Milli,
                     timestamptz_mode,
                     TimeHandlingMode::Milli,
+                    jsonb_handling_mode,
                 )),
                 SinkEncode::Protobuf => {
                     let (descriptor, sid) = crate::schema::protobuf::fetch_descriptor(
@@ -260,7 +261,7 @@ impl MqttSinkWriter {
             },
             _ => {
                 return Err(SinkError::Config(anyhow!(
-                    "Mqtt sink only support append-only mode"
+                    "MQTT sink only supports append-only mode"
                 )))
             }
         };
diff --git a/src/connector/src/sink/nats.rs b/src/connector/src/sink/nats.rs
index 471ce61298417..878cc1bee27d2 100644
--- a/src/connector/src/sink/nats.rs
+++ b/src/connector/src/sink/nats.rs
@@ -28,7 +28,9 @@ use tokio_retry::strategy::{jitter, ExponentialBackoff};
 use tokio_retry::Retry;
 use with_options::WithOptions;
 
-use super::encoder::{DateHandlingMode, TimeHandlingMode, TimestamptzHandlingMode};
+use super::encoder::{
+    DateHandlingMode, JsonbHandlingMode, TimeHandlingMode, TimestamptzHandlingMode,
+};
 use super::utils::chunk_to_json;
 use super::{DummySinkCommitCoordinator, SinkWriterParam};
 use crate::connector_common::NatsCommon;
@@ -77,7 +79,7 @@ impl NatsConfig {
             .map_err(|e| SinkError::Config(anyhow!(e)))?;
         if config.r#type != SINK_TYPE_APPEND_ONLY {
             Err(SinkError::Config(anyhow!(
-                "Nats sink only support append-only mode"
+                "NATS sink only supports append-only mode"
             )))
         } else {
             Ok(config)
@@ -115,7 +117,7 @@ impl Sink for NatsSink {
     async fn validate(&self) -> Result<()> {
         if !self.is_append_only {
             return Err(SinkError::Nats(anyhow!(
-                "Nats sink only support append-only mode"
+                "NATS sink only supports append-only mode"
             )));
         }
         let _client = (self.config.common.build_client().await)
@@ -151,6 +153,7 @@ impl NatsSinkWriter {
                 TimestampHandlingMode::Milli,
                 TimestamptzHandlingMode::UtcWithoutSuffix,
                 TimeHandlingMode::Milli,
+                JsonbHandlingMode::String,
             ),
         })
     }
@@ -164,7 +167,7 @@ impl AsyncTruncateSinkWriter for NatsSinkWriter {
         chunk: StreamChunk,
         mut add_future: DeliveryFutureManagerAddFuture<'a, Self::DeliveryFuture>,
     ) -> Result<()> {
-        let mut data = chunk_to_json(chunk, &self.json_encoder).unwrap();
+        let mut data = chunk_to_json(chunk, &self.json_encoder)?;
         for item in &mut data {
             let publish_ack_future = Retry::spawn(
                 ExponentialBackoff::from_millis(100).map(jitter).take(3),
diff --git a/src/connector/src/sink/pulsar.rs b/src/connector/src/sink/pulsar.rs
index 3f016ad94946d..a92d5b16f85e3 100644
--- a/src/connector/src/sink/pulsar.rs
+++ b/src/connector/src/sink/pulsar.rs
@@ -235,15 +235,20 @@ struct PulsarPayloadWriter<'w> {
     add_future: DeliveryFutureManagerAddFuture<'w, PulsarDeliveryFuture>,
 }
 
-pub type PulsarDeliveryFuture = impl TryFuture<Ok = (), Error = SinkError> + Unpin + 'static;
-
-fn may_delivery_future(future: SendFuture) -> PulsarDeliveryFuture {
-    future.map(|result| {
-        result
-            .map(|_| ())
-            .map_err(|e: pulsar::Error| SinkError::Pulsar(anyhow!(e)))
-    })
+mod opaque_type {
+    use super::*;
+    pub type PulsarDeliveryFuture = impl TryFuture<Ok = (), Error = SinkError> + Unpin + 'static;
+
+    pub(super) fn may_delivery_future(future: SendFuture) -> PulsarDeliveryFuture {
+        future.map(|result| {
+            result
+                .map(|_| ())
+                .map_err(|e: pulsar::Error| SinkError::Pulsar(anyhow!(e)))
+        })
+    }
 }
+use opaque_type::may_delivery_future;
+pub use opaque_type::PulsarDeliveryFuture;
 
 impl PulsarSinkWriter {
     pub async fn new(
diff --git a/src/connector/src/sink/redis.rs b/src/connector/src/sink/redis.rs
index 9d6a33d5131a4..49207e668e41b 100644
--- a/src/connector/src/sink/redis.rs
+++ b/src/connector/src/sink/redis.rs
@@ -233,12 +233,12 @@ impl Sink for RedisSink {
         ) {
             let key_format = self.format_desc.options.get(KEY_FORMAT).ok_or_else(|| {
                 SinkError::Config(anyhow!(
-                    "Cannot find 'key_format',please set it or use JSON"
+                    "Cannot find 'key_format', please set it or use JSON"
                 ))
             })?;
             let value_format = self.format_desc.options.get(VALUE_FORMAT).ok_or_else(|| {
                 SinkError::Config(anyhow!(
-                    "Cannot find 'value_format',please set it or use JSON"
+                    "Cannot find 'value_format', please set it or use JSON"
                 ))
             })?;
             TemplateEncoder::check_string_format(key_format, &pk_set)?;
@@ -299,7 +299,7 @@ impl FormattedSink for RedisSinkPayloadWriter {
     type V = Vec<u8>;
 
     async fn write_one(&mut self, k: Option<Self::K>, v: Option<Self::V>) -> Result<()> {
-        let k = k.unwrap();
+        let k = k.ok_or_else(|| SinkError::Redis("The redis key cannot be null".to_string()))?;
         match v {
             Some(v) => self.pipe.set(k, v),
             None => self.pipe.del(k),
@@ -410,6 +410,7 @@ mod test {
             format: SinkFormat::AppendOnly,
             encode: SinkEncode::Json,
             options: BTreeMap::default(),
+            secret_refs: BTreeMap::default(),
             key_encode: None,
         };
 
@@ -487,6 +488,7 @@ mod test {
             format: SinkFormat::AppendOnly,
             encode: SinkEncode::Template,
             options: btree_map,
+            secret_refs: Default::default(),
             key_encode: None,
         };
 
diff --git a/src/connector/src/sink/remote.rs b/src/connector/src/sink/remote.rs
index d0edded78d6db..23ea54944f11c 100644
--- a/src/connector/src/sink/remote.rs
+++ b/src/connector/src/sink/remote.rs
@@ -169,7 +169,7 @@ async fn validate_remote_sink(param: &SinkParam, sink_name: &str) -> ConnectorRe
         && param.downstream_pk.len() > 1
         && !param.properties.contains_key(ES_OPTION_DELIMITER)
     {
-        bail!("Es sink only support single pk or pk with delimiter option");
+        bail!("Es sink only supports single pk or pk with delimiter option");
     }
     // FIXME: support struct and array in stream sink
     param.columns.iter().map(|col| {
@@ -194,7 +194,7 @@ async fn validate_remote_sink(param: &SinkParam, sink_name: &str) -> ConnectorRe
                     Ok(())
                 } else{
                     Err(SinkError::Remote(anyhow!(
-                        "Remote sink only support list<int16, int32, int64, float, double, varchar>, got {:?}: {:?}",
+                        "Remote sink only supports list<int16, int32, int64, float, double, varchar>, got {:?}: {:?}",
                         col.name,
                         col.data_type,
                     )))
@@ -205,7 +205,7 @@ async fn validate_remote_sink(param: &SinkParam, sink_name: &str) -> ConnectorRe
                     Ok(())
                 }else{
                     Err(SinkError::Remote(anyhow!(
-                        "Only Es sink support struct, got {:?}: {:?}",
+                        "Only Es sink supports struct, got {:?}: {:?}",
                         col.name,
                         col.data_type,
                     )))
@@ -347,7 +347,7 @@ impl LogSinker for RemoteLogSinker {
                 })?;
                 if sent_offset != persisted_offset {
                     bail!(
-                        "new response offset {:?} not match the buffer offset {:?}",
+                        "new response offset {:?} does not match the buffer offset {:?}",
                         persisted_offset,
                         sent_offset
                     );
@@ -535,7 +535,7 @@ impl<R: RemoteSinkTrait> Sink for CoordinatedRemoteSink<R> {
             self.param.clone(),
             writer_param.vnode_bitmap.ok_or_else(|| {
                 SinkError::Remote(anyhow!(
-                    "sink needs coordination should not have singleton input"
+                    "sink needs coordination and should not have singleton input"
                 ))
             })?,
             CoordinatedRemoteSinkWriter::new(self.param.clone(), writer_param.sink_metrics.clone())
diff --git a/src/connector/src/sink/snowflake.rs b/src/connector/src/sink/snowflake.rs
index f08519e3e8da6..6c3cc291f58e2 100644
--- a/src/connector/src/sink/snowflake.rs
+++ b/src/connector/src/sink/snowflake.rs
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::{BTreeMap, HashMap};
+use std::collections::BTreeMap;
 use std::fmt::Write;
 use std::sync::Arc;
 
@@ -20,9 +20,13 @@ use anyhow::{anyhow, Context};
 use async_trait::async_trait;
 use bytes::{Bytes, BytesMut};
 use risingwave_common::array::{Op, StreamChunk};
-use risingwave_common::buffer::Bitmap;
+use risingwave_common::bitmap::Bitmap;
 use risingwave_common::catalog::Schema;
-use risingwave_object_store::object::{ObjectStore, OpendalStreamingUploader, StreamingUploader};
+use risingwave_common::config::ObjectStoreConfig;
+use risingwave_object_store::object::object_metrics::GLOBAL_OBJECT_STORE_METRICS;
+use risingwave_object_store::object::{
+    ObjectStore, OpendalObjectStore, OpendalStreamingUploader, StreamingUploader,
+};
 use serde::Deserialize;
 use serde_json::Value;
 use serde_with::serde_as;
@@ -30,72 +34,46 @@ use uuid::Uuid;
 use with_options::WithOptions;
 
 use super::encoder::{
-    JsonEncoder, RowEncoder, TimeHandlingMode, TimestampHandlingMode, TimestamptzHandlingMode,
+    JsonEncoder, JsonbHandlingMode, RowEncoder, TimeHandlingMode, TimestampHandlingMode,
+    TimestamptzHandlingMode,
 };
-use super::snowflake_connector::{generate_s3_file_name, SnowflakeHttpClient, SnowflakeS3Client};
 use super::writer::LogSinkerOf;
 use super::{SinkError, SinkParam};
 use crate::sink::writer::SinkWriterExt;
 use crate::sink::{DummySinkCommitCoordinator, Result, Sink, SinkWriter, SinkWriterParam};
 
 pub const SNOWFLAKE_SINK: &str = "snowflake";
+const S3_INTERMEDIATE_FILE_NAME: &str = "RW_SNOWFLAKE_S3_SINK_FILE";
 
-#[derive(Deserialize, Debug, Clone, WithOptions)]
+#[derive(Debug, Clone, Deserialize, WithOptions)]
 pub struct SnowflakeCommon {
-    /// The snowflake database used for sinking
-    #[serde(rename = "snowflake.database")]
-    pub database: String,
-
-    /// The corresponding schema where sink table exists
-    #[serde(rename = "snowflake.schema")]
-    pub schema: String,
-
-    /// The created pipe object, will be used as `insertFiles` target
-    #[serde(rename = "snowflake.pipe")]
-    pub pipe: String,
-
-    /// The unique, snowflake provided `account_identifier`
-    /// NOTE: please use the form `<orgname>-<account_name>`
-    /// For detailed guidance, reference: <https://docs.snowflake.com/en/user-guide/admin-account-identifier>
-    #[serde(rename = "snowflake.account_identifier")]
-    pub account_identifier: String,
-
-    /// The user that owns the table to be sinked
-    /// NOTE: the user should've been granted corresponding *role*
-    /// reference: <https://docs.snowflake.com/en/sql-reference/sql/grant-role>
-    #[serde(rename = "snowflake.user")]
-    pub user: String,
-
-    /// The public key fingerprint used when generating custom `jwt_token`
-    /// reference: <https://docs.snowflake.com/en/developer-guide/sql-api/authenticating>
-    #[serde(rename = "snowflake.rsa_public_key_fp")]
-    pub rsa_public_key_fp: String,
-
-    /// The rsa pem key *without* encryption
-    #[serde(rename = "snowflake.private_key")]
-    pub private_key: String,
-
     /// The s3 bucket where intermediate sink files will be stored
-    #[serde(rename = "snowflake.s3_bucket")]
+    #[serde(rename = "snowflake.s3_bucket", alias = "s3.bucket_name")]
     pub s3_bucket: String,
 
     /// The optional s3 path to be specified
     /// the actual file location would be `s3://<s3_bucket>/<s3_path>/<rw_auto_gen_intermediate_file_name>`
     /// if this field is specified by user(s)
     /// otherwise it would be `s3://<s3_bucket>/<rw_auto_gen_intermediate_file_name>`
-    #[serde(rename = "snowflake.s3_path")]
+    #[serde(rename = "snowflake.s3_path", alias = "s3.path")]
     pub s3_path: Option<String>,
 
     /// s3 credentials
-    #[serde(rename = "snowflake.aws_access_key_id")]
+    #[serde(
+        rename = "snowflake.aws_access_key_id",
+        alias = "s3.credentials.access"
+    )]
     pub aws_access_key_id: String,
 
     /// s3 credentials
-    #[serde(rename = "snowflake.aws_secret_access_key")]
+    #[serde(
+        rename = "snowflake.aws_secret_access_key",
+        alias = "s3.credentials.secret"
+    )]
     pub aws_secret_access_key: String,
 
     /// The s3 region, e.g., us-east-2
-    #[serde(rename = "snowflake.aws_region")]
+    #[serde(rename = "snowflake.aws_region", alias = "s3.region_name")]
     pub aws_region: String,
 }
 
@@ -173,8 +151,6 @@ pub struct SnowflakeSinkWriter {
     pk_indices: Vec<usize>,
     #[expect(dead_code)]
     is_append_only: bool,
-    /// the client used to send `insertFiles` post request
-    http_client: SnowflakeHttpClient,
     /// the client to insert file to external storage (i.e., s3)
     s3_client: SnowflakeS3Client,
     row_encoder: JsonEncoder,
@@ -187,7 +163,7 @@ pub struct SnowflakeSinkWriter {
     /// note: the option here *implicitly* indicates whether we have at
     /// least call `streaming_upload` once during this epoch,
     /// which is mainly used to prevent uploading empty data.
-    streaming_uploader: Option<(OpendalStreamingUploader, String)>,
+    streaming_uploader: Option<OpendalStreamingUploader>,
 }
 
 impl SnowflakeSinkWriter {
@@ -197,18 +173,6 @@ impl SnowflakeSinkWriter {
         pk_indices: Vec<usize>,
         is_append_only: bool,
     ) -> Result<Self> {
-        let http_client = SnowflakeHttpClient::new(
-            config.common.account_identifier.clone(),
-            config.common.user.clone(),
-            config.common.database.clone(),
-            config.common.schema.clone(),
-            config.common.pipe.clone(),
-            config.common.rsa_public_key_fp.clone(),
-            config.common.private_key.clone(),
-            HashMap::new(),
-            config.common.s3_path.clone(),
-        );
-
         let s3_client = SnowflakeS3Client::new(
             config.common.s3_bucket.clone(),
             config.common.s3_path.clone(),
@@ -222,7 +186,6 @@ impl SnowflakeSinkWriter {
             schema: schema.clone(),
             pk_indices,
             is_append_only,
-            http_client,
             s3_client,
             row_encoder: JsonEncoder::new(
                 schema,
@@ -231,6 +194,7 @@ impl SnowflakeSinkWriter {
                 TimestampHandlingMode::String,
                 TimestamptzHandlingMode::UtcString,
                 TimeHandlingMode::String,
+                JsonbHandlingMode::String,
             ),
             // initial value of `epoch` will be set to 0
             epoch: 0,
@@ -245,7 +209,7 @@ impl SnowflakeSinkWriter {
     /// and `streaming_upload` being called the first time.
     /// i.e., lazily initialization of the internal `streaming_uploader`.
     /// plus, this function is *pure*, the `&mut self` here is to make rustc (and tokio) happy.
-    async fn new_streaming_uploader(&mut self) -> Result<(OpendalStreamingUploader, String)> {
+    async fn new_streaming_uploader(&mut self) -> Result<OpendalStreamingUploader> {
         let file_suffix = self.file_suffix();
         let path = generate_s3_file_name(self.s3_client.s3_path(), &file_suffix);
         let uploader = self
@@ -260,12 +224,12 @@ impl SnowflakeSinkWriter {
                 )
             })
             .map_err(SinkError::Snowflake)?;
-        Ok((uploader, file_suffix))
+        Ok(uploader)
     }
 
     /// write data to the current streaming uploader for this epoch.
     async fn streaming_upload(&mut self, data: Bytes) -> Result<()> {
-        let (uploader, _) = match self.streaming_uploader.as_mut() {
+        let uploader = match self.streaming_uploader.as_mut() {
             Some(s) => s,
             None => {
                 assert!(
@@ -286,18 +250,18 @@ impl SnowflakeSinkWriter {
 
     /// finalize streaming upload for this epoch.
     /// ensure all the data has been properly uploaded to intermediate s3.
-    async fn finish_streaming_upload(&mut self) -> Result<Option<String>> {
+    async fn finish_streaming_upload(&mut self) -> Result<()> {
         let uploader = std::mem::take(&mut self.streaming_uploader);
-        let Some((uploader, file_suffix)) = uploader else {
+        let Some(uploader) = uploader else {
             // there is no data to be uploaded for this epoch
-            return Ok(None);
+            return Ok(());
         };
         uploader
             .finish()
             .await
             .context("failed to finish streaming upload to s3")
             .map_err(SinkError::Snowflake)?;
-        Ok(Some(file_suffix))
+        Ok(())
     }
 
     async fn append_only(&mut self, chunk: StreamChunk) -> Result<()> {
@@ -344,13 +308,7 @@ impl SnowflakeSinkWriter {
         // note that after `finish_streaming_upload`, do *not* interact with
         // `streaming_uploader` until new data comes in at next epoch,
         // since the ownership has been taken in this method, and `None` will be left.
-        let Some(file_suffix) = self.finish_streaming_upload().await? else {
-            // represents there is no data to be uploaded for this epoch
-            return Ok(());
-        };
-        // trigger `insertFiles` post request to snowflake
-        self.http_client.send_request(&file_suffix).await?;
-        Ok(())
+        self.finish_streaming_upload().await
     }
 }
 
@@ -384,3 +342,57 @@ impl SinkWriter for SnowflakeSinkWriter {
         Ok(())
     }
 }
+
+/// The helper function to generate the *global unique* s3 file name.
+pub(crate) fn generate_s3_file_name(s3_path: Option<&str>, suffix: &str) -> String {
+    match s3_path {
+        Some(path) => format!("{}/{}_{}", path, S3_INTERMEDIATE_FILE_NAME, suffix),
+        None => format!("{}_{}", S3_INTERMEDIATE_FILE_NAME, suffix),
+    }
+}
+
+/// todo: refactor this part after s3 sink is available
+pub struct SnowflakeS3Client {
+    #[expect(dead_code)]
+    s3_bucket: String,
+    s3_path: Option<String>,
+    pub opendal_s3_engine: OpendalObjectStore,
+}
+
+impl SnowflakeS3Client {
+    pub fn new(
+        s3_bucket: String,
+        s3_path: Option<String>,
+        aws_access_key_id: String,
+        aws_secret_access_key: String,
+        aws_region: String,
+    ) -> Result<Self> {
+        // FIXME: we should use the `ObjectStoreConfig` instead of default
+        // just use default configuration here for opendal s3 engine
+        let config = ObjectStoreConfig::default();
+
+        let metrics = Arc::new(GLOBAL_OBJECT_STORE_METRICS.clone());
+
+        // create the s3 engine for streaming upload to the intermediate s3 bucket
+        let opendal_s3_engine = OpendalObjectStore::new_s3_engine_with_credentials(
+            &s3_bucket,
+            Arc::new(config),
+            metrics,
+            &aws_access_key_id,
+            &aws_secret_access_key,
+            &aws_region,
+        )
+        .context("failed to create opendal s3 engine")
+        .map_err(SinkError::Snowflake)?;
+
+        Ok(Self {
+            s3_bucket,
+            s3_path,
+            opendal_s3_engine,
+        })
+    }
+
+    pub fn s3_path(&self) -> Option<&str> {
+        self.s3_path.as_deref()
+    }
+}
diff --git a/src/connector/src/sink/snowflake_connector.rs b/src/connector/src/sink/snowflake_connector.rs
deleted file mode 100644
index bfd2458900294..0000000000000
--- a/src/connector/src/sink/snowflake_connector.rs
+++ /dev/null
@@ -1,225 +0,0 @@
-// Copyright 2024 RisingWave Labs
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::collections::HashMap;
-use std::sync::Arc;
-use std::time::{SystemTime, UNIX_EPOCH};
-
-use anyhow::{anyhow, Context};
-use jsonwebtoken::{encode, Algorithm, EncodingKey, Header};
-use object_metrics::GLOBAL_OBJECT_STORE_METRICS;
-use reqwest::{header, Client, RequestBuilder, StatusCode};
-use risingwave_object_store::object::*;
-use serde::{Deserialize, Serialize};
-
-use super::doris_starrocks_connector::POOL_IDLE_TIMEOUT;
-use super::{Result, SinkError};
-
-const SNOWFLAKE_HOST_ADDR: &str = "snowflakecomputing.com";
-const SNOWFLAKE_REQUEST_ID: &str = "RW_SNOWFLAKE_SINK";
-const S3_INTERMEDIATE_FILE_NAME: &str = "RW_SNOWFLAKE_S3_SINK_FILE";
-
-/// The helper function to generate the *global unique* s3 file name.
-pub(crate) fn generate_s3_file_name(s3_path: Option<&str>, suffix: &str) -> String {
-    match s3_path {
-        Some(path) => format!("{}/{}_{}", path, S3_INTERMEDIATE_FILE_NAME, suffix),
-        None => format!("{}_{}", S3_INTERMEDIATE_FILE_NAME, suffix),
-    }
-}
-
-/// Claims is used when constructing `jwt_token`
-/// with payload specified.
-/// reference: <https://docs.snowflake.com/en/developer-guide/sql-api/authenticating>
-#[derive(Debug, Serialize, Deserialize)]
-struct Claims {
-    iss: String,
-    sub: String,
-    iat: usize,
-    exp: usize,
-}
-
-#[derive(Debug)]
-pub struct SnowflakeHttpClient {
-    url: String,
-    rsa_public_key_fp: String,
-    account: String,
-    user: String,
-    private_key: String,
-    #[expect(dead_code)]
-    header: HashMap<String, String>,
-    s3_path: Option<String>,
-}
-
-impl SnowflakeHttpClient {
-    pub fn new(
-        account: String,
-        user: String,
-        db: String,
-        schema: String,
-        pipe: String,
-        rsa_public_key_fp: String,
-        private_key: String,
-        header: HashMap<String, String>,
-        s3_path: Option<String>,
-    ) -> Self {
-        // todo: ensure if we need user to *explicitly* provide the `request_id`
-        // currently it seems that this is not important.
-        // reference to the snowpipe rest api is as below, i.e.,
-        // <https://docs.snowflake.com/en/user-guide/data-load-snowpipe-rest-apis>
-        let url = format!(
-            "https://{}.{}/v1/data/pipes/{}.{}.{}/insertFiles?requestId={}",
-            account.clone(),
-            SNOWFLAKE_HOST_ADDR,
-            db,
-            schema,
-            pipe,
-            SNOWFLAKE_REQUEST_ID
-        );
-
-        Self {
-            url,
-            rsa_public_key_fp,
-            account,
-            user,
-            private_key,
-            header,
-            s3_path,
-        }
-    }
-
-    /// Generate a 59-minutes valid `jwt_token` for authentication of snowflake side
-    /// And please note that we will NOT strictly counting the time interval
-    /// of `jwt_token` expiration.
-    /// Which essentially means that this method should be called *every time* we want
-    /// to send `insertFiles` request to snowflake server.
-    fn generate_jwt_token(&self) -> Result<String> {
-        let header = Header::new(Algorithm::RS256);
-        let now = SystemTime::now()
-            .duration_since(UNIX_EPOCH)
-            .unwrap()
-            .as_secs() as usize;
-        let lifetime = 59 * 60;
-
-        // Ensure the account and username are uppercase
-        let account = self.account.to_uppercase();
-        let user = self.user.to_uppercase();
-
-        // Construct the fully qualified username
-        let qualified_username = format!("{}.{}", account, user);
-
-        let claims = Claims {
-            iss: format!("{}.{}", qualified_username.clone(), self.rsa_public_key_fp),
-            sub: qualified_username,
-            iat: now,
-            exp: now + lifetime,
-        };
-
-        let jwt_token = encode(
-            &header,
-            &claims,
-            &EncodingKey::from_rsa_pem(self.private_key.as_ref())
-                .context("failed to encode from provided rsa pem key")
-                .map_err(SinkError::Snowflake)?,
-        )
-        .context("failed to encode jwt_token")
-        .map_err(SinkError::Snowflake)?;
-        Ok(jwt_token)
-    }
-
-    fn build_request_and_client(&self) -> RequestBuilder {
-        let client = Client::builder()
-            .pool_idle_timeout(POOL_IDLE_TIMEOUT)
-            .build()
-            .unwrap();
-
-        client.post(&self.url)
-    }
-
-    /// NOTE: this function should ONLY be called *after*
-    /// uploading files to remote external staged storage, i.e., AWS S3
-    pub async fn send_request(&self, file_suffix: &str) -> Result<()> {
-        let builder = self.build_request_and_client();
-
-        // Generate the jwt_token
-        let jwt_token = self.generate_jwt_token()?;
-        let builder = builder
-            .header(header::CONTENT_TYPE, "text/plain")
-            .header("Authorization", format!("Bearer {}", jwt_token))
-            .header(
-                "X-Snowflake-Authorization-Token-Type".to_string(),
-                "KEYPAIR_JWT",
-            )
-            .body(generate_s3_file_name(self.s3_path.as_deref(), file_suffix));
-
-        let response = builder
-            .send()
-            .await
-            .map_err(|err| SinkError::Snowflake(anyhow!(err)))?;
-
-        if response.status() != StatusCode::OK {
-            return Err(SinkError::Snowflake(anyhow!(
-                "failed to make http request, error code: {}\ndetailed response: {:#?}",
-                response.status(),
-                response,
-            )));
-        }
-
-        Ok(())
-    }
-}
-
-/// todo: refactor this part after s3 sink is available
-pub struct SnowflakeS3Client {
-    #[expect(dead_code)]
-    s3_bucket: String,
-    s3_path: Option<String>,
-    pub opendal_s3_engine: OpendalObjectStore,
-}
-
-impl SnowflakeS3Client {
-    pub fn new(
-        s3_bucket: String,
-        s3_path: Option<String>,
-        aws_access_key_id: String,
-        aws_secret_access_key: String,
-        aws_region: String,
-    ) -> Result<Self> {
-        // FIXME: we should use the `ObjectStoreConfig` instead of default
-        // just use default configuration here for opendal s3 engine
-        let config = ObjectStoreConfig::default();
-        let metrics = Arc::new(GLOBAL_OBJECT_STORE_METRICS.clone());
-
-        // create the s3 engine for streaming upload to the intermediate s3 bucket
-        let opendal_s3_engine = OpendalObjectStore::new_s3_engine_with_credentials(
-            &s3_bucket,
-            Arc::new(config),
-            metrics,
-            &aws_access_key_id,
-            &aws_secret_access_key,
-            &aws_region,
-        )
-        .context("failed to create opendal s3 engine")
-        .map_err(SinkError::Snowflake)?;
-
-        Ok(Self {
-            s3_bucket,
-            s3_path,
-            opendal_s3_engine,
-        })
-    }
-
-    pub fn s3_path(&self) -> Option<&str> {
-        self.s3_path.as_deref()
-    }
-}
diff --git a/src/connector/src/sink/sqlserver.rs b/src/connector/src/sink/sqlserver.rs
index acdad3c47627e..0295d3d3de9ad 100644
--- a/src/connector/src/sink/sqlserver.rs
+++ b/src/connector/src/sink/sqlserver.rs
@@ -15,10 +15,10 @@
 use std::collections::{BTreeMap, HashMap};
 use std::sync::Arc;
 
-use anyhow::anyhow;
+use anyhow::{anyhow, Context};
 use async_trait::async_trait;
 use risingwave_common::array::{Op, RowRef, StreamChunk};
-use risingwave_common::buffer::Bitmap;
+use risingwave_common::bitmap::Bitmap;
 use risingwave_common::catalog::Schema;
 use risingwave_common::row::{OwnedRow, Row};
 use risingwave_common::types::{DataType, Decimal};
@@ -144,6 +144,10 @@ impl Sink for SqlServerSink {
     const SINK_NAME: &'static str = SQLSERVER_SINK;
 
     async fn validate(&self) -> Result<()> {
+        risingwave_common::license::Feature::SqlServerSink
+            .check_available()
+            .map_err(|e| anyhow::anyhow!(e))?;
+
         if !self.is_append_only && self.pk_indices.is_empty() {
             return Err(SinkError::Config(anyhow!(
                 "Primary key not defined for upsert SQL Server sink (please define in `primary_key` field)")));
@@ -155,7 +159,7 @@ impl Sink for SqlServerSink {
 
         // Query table metadata from SQL Server.
         let mut sql_server_table_metadata = HashMap::new();
-        let mut sql_client = SqlClient::new(&self.config).await?;
+        let mut sql_client = SqlServerClient::new(&self.config).await?;
         let query_table_metadata_error = || {
             SinkError::SqlServer(anyhow!(format!(
                 "SQL Server table {} metadata error",
@@ -177,7 +181,7 @@ WHERE
 ORDER BY
     col.column_id;"#;
         let rows = sql_client
-            .client
+            .inner_client
             .query(QUERY_TABLE_METADATA, &[&self.config.table])
             .await?
             .into_results()
@@ -268,7 +272,7 @@ pub struct SqlServerSinkWriter {
     schema: Schema,
     pk_indices: Vec<usize>,
     is_append_only: bool,
-    sql_client: SqlClient,
+    sql_client: SqlServerClient,
     ops: Vec<SqlOp>,
 }
 
@@ -279,7 +283,7 @@ impl SqlServerSinkWriter {
         pk_indices: Vec<usize>,
         is_append_only: bool,
     ) -> Result<Self> {
-        let sql_client = SqlClient::new(&config).await?;
+        let sql_client = SqlServerClient::new(&config).await?;
         let writer = Self {
             config,
             schema,
@@ -440,7 +444,7 @@ impl SqlServerSinkWriter {
                 }
             }
         }
-        query.execute(&mut self.sql_client.client).await?;
+        query.execute(&mut self.sql_client.inner_client).await?;
         Ok(())
     }
 }
@@ -491,11 +495,12 @@ impl SinkWriter for SqlServerSinkWriter {
     }
 }
 
-struct SqlClient {
-    client: Client<tokio_util::compat::Compat<TcpStream>>,
+#[derive(Debug)]
+pub struct SqlServerClient {
+    pub inner_client: Client<tokio_util::compat::Compat<TcpStream>>,
 }
 
-impl SqlClient {
+impl SqlServerClient {
     async fn new(msconfig: &SqlServerConfig) -> Result<Self> {
         let mut config = Config::new();
         config.host(&msconfig.host);
@@ -503,11 +508,43 @@ impl SqlClient {
         config.authentication(AuthMethod::sql_server(&msconfig.user, &msconfig.password));
         config.database(&msconfig.database);
         config.trust_cert();
+        Self::new_with_config(config).await
+    }
 
-        let tcp = TcpStream::connect(config.get_addr()).await.unwrap();
-        tcp.set_nodelay(true).unwrap();
-        let client = Client::connect(config, tcp.compat_write()).await?;
-        Ok(Self { client })
+    pub async fn new_with_config(mut config: Config) -> Result<Self> {
+        let tcp = TcpStream::connect(config.get_addr())
+            .await
+            .context("failed to connect to sql server")
+            .map_err(SinkError::SqlServer)?;
+        tcp.set_nodelay(true)
+            .context("failed to setting nodelay when connecting to sql server")
+            .map_err(SinkError::SqlServer)?;
+
+        let client = match Client::connect(config.clone(), tcp.compat_write()).await {
+            // Connection successful.
+            Ok(client) => client,
+            // The server wants us to redirect to a different address
+            Err(tiberius::error::Error::Routing { host, port }) => {
+                config.host(&host);
+                config.port(port);
+                let tcp = TcpStream::connect(config.get_addr())
+                    .await
+                    .context("failed to connect to sql server after routing")
+                    .map_err(SinkError::SqlServer)?;
+                tcp.set_nodelay(true)
+                    .context(
+                        "failed to setting nodelay when connecting to sql server after routing",
+                    )
+                    .map_err(SinkError::SqlServer)?;
+                // we should not have more than one redirect, so we'll short-circuit here.
+                Client::connect(config, tcp.compat_write()).await?
+            }
+            Err(e) => return Err(e.into()),
+        };
+
+        Ok(Self {
+            inner_client: client,
+        })
     }
 }
 
diff --git a/src/connector/src/sink/starrocks.rs b/src/connector/src/sink/starrocks.rs
index 64dadf0b89866..e5881ee9f747e 100644
--- a/src/connector/src/sink/starrocks.rs
+++ b/src/connector/src/sink/starrocks.rs
@@ -22,7 +22,7 @@ use bytes::Bytes;
 use mysql_async::prelude::Queryable;
 use mysql_async::Opts;
 use risingwave_common::array::{Op, StreamChunk};
-use risingwave_common::buffer::Bitmap;
+use risingwave_common::bitmap::Bitmap;
 use risingwave_common::catalog::Schema;
 use risingwave_common::session_config::sink_decouple::SinkDecouple;
 use risingwave_common::types::DataType;
@@ -340,7 +340,7 @@ impl Sink for StarrocksSink {
             self.param.clone(),
             writer_param.vnode_bitmap.ok_or_else(|| {
                 SinkError::Remote(anyhow!(
-                    "sink needs coordination should not have singleton input"
+                    "sink needs coordination and should not have singleton input"
                 ))
             })?,
             inner,
diff --git a/src/connector/src/sink/writer.rs b/src/connector/src/sink/writer.rs
index 00917f26c4163..7057fa8c8b510 100644
--- a/src/connector/src/sink/writer.rs
+++ b/src/connector/src/sink/writer.rs
@@ -21,7 +21,7 @@ use async_trait::async_trait;
 use futures::future::{select, Either};
 use futures::TryFuture;
 use risingwave_common::array::StreamChunk;
-use risingwave_common::buffer::Bitmap;
+use risingwave_common::bitmap::Bitmap;
 use rw_futures_util::drop_either_future;
 
 use crate::sink::encoder::SerTo;
diff --git a/src/connector/src/source/base.rs b/src/connector/src/source/base.rs
index 1f2444a0db4e7..dc00fee24200e 100644
--- a/src/connector/src/source/base.rs
+++ b/src/connector/src/source/base.rs
@@ -26,6 +26,7 @@ use itertools::Itertools;
 use risingwave_common::array::StreamChunk;
 use risingwave_common::bail;
 use risingwave_common::catalog::TableId;
+use risingwave_common::secret::LocalSecretManager;
 use risingwave_common::types::{JsonbVal, Scalar};
 use risingwave_pb::catalog::{PbSource, PbStreamSourceInfo};
 use risingwave_pb::plan_common::ExternalTableDesc;
@@ -44,11 +45,11 @@ use crate::error::ConnectorResult as Result;
 use crate::parser::ParserConfig;
 use crate::source::filesystem::FsPageItem;
 use crate::source::monitor::EnumeratorMetrics;
-use crate::source::SplitImpl::{CitusCdc, MongodbCdc, MysqlCdc, PostgresCdc};
+use crate::source::SplitImpl::{CitusCdc, MongodbCdc, MysqlCdc, PostgresCdc, SqlServerCdc};
 use crate::with_options::WithOptions;
 use crate::{
     dispatch_source_prop, dispatch_split_impl, for_all_sources, impl_connector_properties,
-    impl_split, match_source_name_str,
+    impl_split, match_source_name_str, WithOptionsSecResolved,
 };
 
 const SPLIT_TYPE_FIELD: &str = "split_type";
@@ -243,6 +244,7 @@ pub enum SourceEncode {
     Protobuf,
     Json,
     Bytes,
+    Parquet,
 }
 
 #[derive(Clone, Copy, Debug, PartialEq, Eq, Default)]
@@ -295,6 +297,9 @@ pub fn extract_source_struct(info: &PbStreamSourceInfo) -> Result<SourceStruct>
         (PbFormatType::Maxwell, PbEncodeType::Json) => (SourceFormat::Maxwell, SourceEncode::Json),
         (PbFormatType::Canal, PbEncodeType::Json) => (SourceFormat::Canal, SourceEncode::Json),
         (PbFormatType::Plain, PbEncodeType::Csv) => (SourceFormat::Plain, SourceEncode::Csv),
+        (PbFormatType::Plain, PbEncodeType::Parquet) => {
+            (SourceFormat::Plain, SourceEncode::Parquet)
+        }
         (PbFormatType::Native, PbEncodeType::Native) => {
             (SourceFormat::Native, SourceEncode::Native)
         }
@@ -383,16 +388,19 @@ impl ConnectorProperties {
     /// `deny_unknown_fields`: Since `WITH` options are persisted in meta, we do not deny unknown fields when restoring from
     /// existing data to avoid breaking backwards compatibility. We only deny unknown fields when creating new sources.
     pub fn extract(
-        mut with_properties: BTreeMap<String, String>,
+        with_properties: WithOptionsSecResolved,
         deny_unknown_fields: bool,
     ) -> Result<Self> {
-        let connector = with_properties
+        let (options, secret_refs) = with_properties.into_parts();
+        let mut options_with_secret =
+            LocalSecretManager::global().fill_secrets(options, secret_refs)?;
+        let connector = options_with_secret
             .remove(UPSTREAM_SOURCE_KEY)
             .ok_or_else(|| anyhow!("Must specify 'connector' in WITH clause"))?;
         match_source_name_str!(
             connector.to_lowercase().as_str(),
             PropType,
-            PropType::try_from_btreemap(with_properties, deny_unknown_fields)
+            PropType::try_from_btreemap(options_with_secret, deny_unknown_fields)
                 .map(ConnectorProperties::from),
             |other| bail!("connector '{}' is not supported", other)
         )
@@ -464,7 +472,7 @@ impl SplitImpl {
     pub fn is_cdc_split(&self) -> bool {
         matches!(
             self,
-            MysqlCdc(_) | PostgresCdc(_) | MongodbCdc(_) | CitusCdc(_)
+            MysqlCdc(_) | PostgresCdc(_) | MongodbCdc(_) | CitusCdc(_) | SqlServerCdc(_)
         )
     }
 
@@ -475,6 +483,7 @@ impl SplitImpl {
             PostgresCdc(split) => split.start_offset().clone().unwrap_or_default(),
             MongodbCdc(split) => split.start_offset().clone().unwrap_or_default(),
             CitusCdc(split) => split.start_offset().clone().unwrap_or_default(),
+            SqlServerCdc(split) => split.start_offset().clone().unwrap_or_default(),
             _ => unreachable!("get_cdc_split_offset() is only for cdc split"),
         }
     }
@@ -686,7 +695,9 @@ mod tests {
             "nexmark.split.num" => "1",
         ));
 
-        let props = ConnectorProperties::extract(props, true).unwrap();
+        let props =
+            ConnectorProperties::extract(WithOptionsSecResolved::without_secrets(props), true)
+                .unwrap();
 
         if let ConnectorProperties::Nexmark(props) = props {
             assert_eq!(props.table_type, Some(EventType::Person));
@@ -706,7 +717,9 @@ mod tests {
             "broker.rewrite.endpoints" => r#"{"b-1:9092":"dns-1", "b-2:9092":"dns-2"}"#,
         ));
 
-        let props = ConnectorProperties::extract(props, true).unwrap();
+        let props =
+            ConnectorProperties::extract(WithOptionsSecResolved::without_secrets(props), true)
+                .unwrap();
         if let ConnectorProperties::Kafka(k) = props {
             let btreemap = btreemap! {
                 "b-1:9092".to_string() => "dns-1".to_string(),
@@ -741,7 +754,11 @@ mod tests {
             "table.name" => "orders",
         ));
 
-        let conn_props = ConnectorProperties::extract(user_props_mysql, true).unwrap();
+        let conn_props = ConnectorProperties::extract(
+            WithOptionsSecResolved::without_secrets(user_props_mysql),
+            true,
+        )
+        .unwrap();
         if let ConnectorProperties::MysqlCdc(c) = conn_props {
             assert_eq!(c.properties.get("database.hostname").unwrap(), "127.0.0.1");
             assert_eq!(c.properties.get("database.port").unwrap(), "3306");
@@ -753,7 +770,11 @@ mod tests {
             panic!("extract cdc config failed");
         }
 
-        let conn_props = ConnectorProperties::extract(user_props_postgres, true).unwrap();
+        let conn_props = ConnectorProperties::extract(
+            WithOptionsSecResolved::without_secrets(user_props_postgres),
+            true,
+        )
+        .unwrap();
         if let ConnectorProperties::PostgresCdc(c) = conn_props {
             assert_eq!(c.properties.get("database.hostname").unwrap(), "127.0.0.1");
             assert_eq!(c.properties.get("database.port").unwrap(), "5432");
diff --git a/src/connector/src/source/cdc/enumerator/mod.rs b/src/connector/src/source/cdc/enumerator/mod.rs
index 1c0237b0654c7..ae8241a769108 100644
--- a/src/connector/src/source/cdc/enumerator/mod.rs
+++ b/src/connector/src/source/cdc/enumerator/mod.rs
@@ -28,7 +28,7 @@ use risingwave_pb::connector_service::{SourceType, ValidateSourceRequest, Valida
 use crate::error::ConnectorResult;
 use crate::source::cdc::{
     table_schema_exclude_additional_columns, CdcProperties, CdcSourceTypeTrait, Citus,
-    DebeziumCdcSplit, Mongodb, Mysql, Postgres,
+    DebeziumCdcSplit, Mongodb, Mysql, Postgres, SqlServer,
 };
 use crate::source::{SourceEnumeratorContextRef, SplitEnumerator};
 
@@ -185,3 +185,15 @@ impl ListCdcSplits for DebeziumSplitEnumerator<Mongodb> {
         )]
     }
 }
+
+impl ListCdcSplits for DebeziumSplitEnumerator<SqlServer> {
+    type CdcSourceType = SqlServer;
+
+    fn list_cdc_splits(&mut self) -> Vec<DebeziumCdcSplit<Self::CdcSourceType>> {
+        vec![DebeziumCdcSplit::<Self::CdcSourceType>::new(
+            self.source_id,
+            None,
+            None,
+        )]
+    }
+}
diff --git a/src/connector/src/source/cdc/external/mod.rs b/src/connector/src/source/cdc/external/mod.rs
index 528e14bd60229..be1c891b8d078 100644
--- a/src/connector/src/source/cdc/external/mod.rs
+++ b/src/connector/src/source/cdc/external/mod.rs
@@ -14,6 +14,7 @@
 
 pub mod mock_external_table;
 pub mod postgres;
+pub mod sql_server;
 
 #[cfg(not(madsim))]
 mod maybe_tls_connector;
@@ -29,6 +30,8 @@ use futures_async_stream::try_stream;
 use risingwave_common::bail;
 use risingwave_common::catalog::{ColumnDesc, Schema};
 use risingwave_common::row::OwnedRow;
+use risingwave_common::secret::LocalSecretManager;
+use risingwave_pb::secret::PbSecretRef;
 use serde_derive::{Deserialize, Serialize};
 
 use crate::error::{ConnectorError, ConnectorResult};
@@ -40,6 +43,9 @@ use crate::source::cdc::external::mysql::{
 use crate::source::cdc::external::postgres::{
     PostgresExternalTable, PostgresExternalTableReader, PostgresOffset,
 };
+use crate::source::cdc::external::sql_server::{
+    SqlServerExternalTable, SqlServerExternalTableReader, SqlServerOffset,
+};
 use crate::source::cdc::CdcSourceType;
 use crate::WithPropertiesExt;
 
@@ -48,6 +54,7 @@ pub enum CdcTableType {
     Undefined,
     MySql,
     Postgres,
+    SqlServer,
     Citus,
 }
 
@@ -58,27 +65,41 @@ impl CdcTableType {
             "mysql-cdc" => Self::MySql,
             "postgres-cdc" => Self::Postgres,
             "citus-cdc" => Self::Citus,
+            "sqlserver-cdc" => Self::SqlServer,
             _ => Self::Undefined,
         }
     }
 
     pub fn can_backfill(&self) -> bool {
+        matches!(self, Self::MySql | Self::Postgres | Self::SqlServer)
+    }
+
+    pub fn enable_transaction_metadata(&self) -> bool {
+        // In Debezium, transactional metadata cause delay of the newest events, as the `END` message is never sent unless a new transaction starts.
+        // So we only allow transactional metadata for MySQL and Postgres.
+        // See more in https://debezium.io/documentation/reference/2.6/connectors/sqlserver.html#sqlserver-transaction-metadata
         matches!(self, Self::MySql | Self::Postgres)
     }
 
+    pub fn shareable_only(&self) -> bool {
+        matches!(self, Self::SqlServer)
+    }
+
     pub async fn create_table_reader(
         &self,
         config: ExternalTableConfig,
         schema: Schema,
         pk_indices: Vec<usize>,
-        scan_limit: u32,
     ) -> ConnectorResult<ExternalTableReaderImpl> {
         match self {
             Self::MySql => Ok(ExternalTableReaderImpl::MySql(
                 MySqlExternalTableReader::new(config, schema).await?,
             )),
             Self::Postgres => Ok(ExternalTableReaderImpl::Postgres(
-                PostgresExternalTableReader::new(config, schema, pk_indices, scan_limit).await?,
+                PostgresExternalTableReader::new(config, schema, pk_indices).await?,
+            )),
+            Self::SqlServer => Ok(ExternalTableReaderImpl::SqlServer(
+                SqlServerExternalTableReader::new(config, schema, pk_indices).await?,
             )),
             _ => bail!("invalid external table type: {:?}", *self),
         }
@@ -97,7 +118,7 @@ pub const SCHEMA_NAME_KEY: &str = "schema.name";
 pub const DATABASE_NAME_KEY: &str = "database.name";
 
 impl SchemaTableName {
-    pub fn from_properties(properties: &HashMap<String, String>) -> Self {
+    pub fn from_properties(properties: &BTreeMap<String, String>) -> Self {
         let table_type = CdcTableType::from_properties(properties);
         let table_name = properties.get(TABLE_NAME_KEY).cloned().unwrap_or_default();
 
@@ -109,6 +130,7 @@ impl SchemaTableName {
             CdcTableType::Postgres | CdcTableType::Citus => {
                 properties.get(SCHEMA_NAME_KEY).cloned().unwrap_or_default()
             }
+            CdcTableType::SqlServer => properties.get(SCHEMA_NAME_KEY).cloned().unwrap_or_default(),
             _ => {
                 unreachable!("invalid external table type: {:?}", table_type);
             }
@@ -125,6 +147,7 @@ impl SchemaTableName {
 pub enum CdcOffset {
     MySql(MySqlOffset),
     Postgres(PostgresOffset),
+    SqlServer(SqlServerOffset),
 }
 
 // Example debezium offset for Postgres:
@@ -168,6 +191,10 @@ pub struct DebeziumSourceOffset {
     #[serde(rename = "txId")]
     pub txid: Option<i64>,
     pub tx_usec: Option<u64>,
+
+    // sql server offset
+    pub commit_lsn: Option<String>,
+    pub change_lsn: Option<String>,
 }
 
 pub type CdcOffsetParseFunc = Box<dyn Fn(&str) -> ConnectorResult<CdcOffset> + Send>;
@@ -187,6 +214,7 @@ pub trait ExternalTableReader {
 pub enum ExternalTableReaderImpl {
     MySql(MySqlExternalTableReader),
     Postgres(PostgresExternalTableReader),
+    SqlServer(SqlServerExternalTableReader),
     Mock(MockExternalTableReader),
 }
 
@@ -215,8 +243,11 @@ pub struct ExternalTableConfig {
 impl ExternalTableConfig {
     pub fn try_from_btreemap(
         connect_properties: BTreeMap<String, String>,
+        secret_refs: BTreeMap<String, PbSecretRef>,
     ) -> ConnectorResult<Self> {
-        let json_value = serde_json::to_value(connect_properties)?;
+        let options_with_secret =
+            LocalSecretManager::global().fill_secrets(connect_properties, secret_refs)?;
+        let json_value = serde_json::to_value(options_with_secret)?;
         let config = serde_json::from_value::<ExternalTableConfig>(json_value)?;
         Ok(config)
     }
@@ -255,6 +286,7 @@ impl ExternalTableReader for ExternalTableReaderImpl {
         match self {
             ExternalTableReaderImpl::MySql(mysql) => mysql.current_cdc_offset().await,
             ExternalTableReaderImpl::Postgres(postgres) => postgres.current_cdc_offset().await,
+            ExternalTableReaderImpl::SqlServer(sql_server) => sql_server.current_cdc_offset().await,
             ExternalTableReaderImpl::Mock(mock) => mock.current_cdc_offset().await,
         }
     }
@@ -277,6 +309,9 @@ impl ExternalTableReaderImpl {
             ExternalTableReaderImpl::Postgres(_) => {
                 PostgresExternalTableReader::get_cdc_offset_parser()
             }
+            ExternalTableReaderImpl::SqlServer(_) => {
+                SqlServerExternalTableReader::get_cdc_offset_parser()
+            }
             ExternalTableReaderImpl::Mock(_) => MockExternalTableReader::get_cdc_offset_parser(),
         }
     }
@@ -296,6 +331,9 @@ impl ExternalTableReaderImpl {
             ExternalTableReaderImpl::Postgres(postgres) => {
                 postgres.snapshot_read(table_name, start_pk, primary_keys, limit)
             }
+            ExternalTableReaderImpl::SqlServer(sql_server) => {
+                sql_server.snapshot_read(table_name, start_pk, primary_keys, limit)
+            }
             ExternalTableReaderImpl::Mock(mock) => {
                 mock.snapshot_read(table_name, start_pk, primary_keys, limit)
             }
@@ -313,6 +351,7 @@ impl ExternalTableReaderImpl {
 pub enum ExternalTableImpl {
     MySql(MySqlExternalTable),
     Postgres(PostgresExternalTable),
+    SqlServer(SqlServerExternalTable),
 }
 
 impl ExternalTableImpl {
@@ -325,6 +364,9 @@ impl ExternalTableImpl {
             CdcSourceType::Postgres => Ok(ExternalTableImpl::Postgres(
                 PostgresExternalTable::connect(config).await?,
             )),
+            CdcSourceType::SqlServer => Ok(ExternalTableImpl::SqlServer(
+                SqlServerExternalTable::connect(config).await?,
+            )),
             _ => Err(anyhow!("Unsupported cdc connector type: {}", config.connector).into()),
         }
     }
@@ -333,6 +375,7 @@ impl ExternalTableImpl {
         match self {
             ExternalTableImpl::MySql(mysql) => mysql.column_descs(),
             ExternalTableImpl::Postgres(postgres) => postgres.column_descs(),
+            ExternalTableImpl::SqlServer(sql_server) => sql_server.column_descs(),
         }
     }
 
@@ -340,6 +383,7 @@ impl ExternalTableImpl {
         match self {
             ExternalTableImpl::MySql(mysql) => mysql.pk_names(),
             ExternalTableImpl::Postgres(postgres) => postgres.pk_names(),
+            ExternalTableImpl::SqlServer(sql_server) => sql_server.pk_names(),
         }
     }
 }
diff --git a/src/connector/src/source/cdc/external/mysql.rs b/src/connector/src/source/cdc/external/mysql.rs
index e5f53720dd6ee..6947ba7a46d6b 100644
--- a/src/connector/src/source/cdc/external/mysql.rs
+++ b/src/connector/src/source/cdc/external/mysql.rs
@@ -106,7 +106,7 @@ impl MySqlExternalTable {
         let mut column_descs = vec![];
         let mut pk_names = vec![];
         for col in columns {
-            let data_type = type_to_rw_type(&col.col_type)?;
+            let data_type = mysql_type_to_rw_type(&col.col_type)?;
             // column name in mysql is case-insensitive, convert to lowercase
             let col_name = col.name.to_lowercase();
             column_descs.push(ColumnDesc::named(
@@ -138,7 +138,56 @@ impl MySqlExternalTable {
     }
 }
 
-fn type_to_rw_type(col_type: &ColumnType) -> ConnectorResult<DataType> {
+pub fn type_name_to_mysql_type(ty_name: &str) -> Option<ColumnType> {
+    macro_rules! column_type {
+        ($($name:literal => $variant:ident),* $(,)?) => {
+            match ty_name.to_lowercase().as_str() {
+                $(
+                    $name => Some(ColumnType::$variant(Default::default())),
+                )*
+                _ => None,
+            }
+        };
+    }
+
+    column_type! {
+        "bit" => Bit,
+        "tinyint" => TinyInt,
+        "smallint" => SmallInt,
+        "mediumint" => MediumInt,
+        "int" => Int,
+        "bigint" => BigInt,
+        "decimal" => Decimal,
+        "float" => Float,
+        "double" => Double,
+        "time" => Time,
+        "datetime" => DateTime,
+        "timestamp" => Timestamp,
+        "char" => Char,
+        "nchar" => NChar,
+        "varchar" => Varchar,
+        "nvarchar" => NVarchar,
+        "binary" => Binary,
+        "varbinary" => Varbinary,
+        "text" => Text,
+        "tinytext" => TinyText,
+        "mediumtext" => MediumText,
+        "longtext" => LongText,
+        "blob" => Blob,
+        "enum" => Enum,
+        "set" => Set,
+        "geometry" => Geometry,
+        "point" => Point,
+        "linestring" => LineString,
+        "polygon" => Polygon,
+        "multipoint" => MultiPoint,
+        "multilinestring" => MultiLineString,
+        "multipolygon" => MultiPolygon,
+        "geometrycollection" => GeometryCollection,
+    }
+}
+
+pub fn mysql_type_to_rw_type(col_type: &ColumnType) -> ConnectorResult<DataType> {
     let dtype = match col_type {
         ColumnType::Serial => DataType::Int32,
         ColumnType::Bit(attr) => {
diff --git a/src/connector/src/source/cdc/external/postgres.rs b/src/connector/src/source/cdc/external/postgres.rs
index 0bbf5dccbe6de..ca0caf46d6125 100644
--- a/src/connector/src/source/cdc/external/postgres.rs
+++ b/src/connector/src/source/cdc/external/postgres.rs
@@ -33,7 +33,7 @@ use sqlx::postgres::{PgConnectOptions, PgSslMode};
 use sqlx::PgPool;
 use thiserror_ext::AsReport;
 use tokio_postgres::types::PgLsn;
-use tokio_postgres::{NoTls, Statement};
+use tokio_postgres::NoTls;
 
 use crate::error::{ConnectorError, ConnectorResult};
 use crate::parser::postgres_row_to_owned_row;
@@ -232,7 +232,7 @@ fn type_to_rw_type(col_type: &ColumnType) -> ConnectorResult<DataType> {
 pub struct PostgresExternalTableReader {
     rw_schema: Schema,
     field_names: String,
-    prepared_scan_stmt: Statement,
+    pk_indices: Vec<usize>,
     client: tokio::sync::Mutex<tokio_postgres::Client>,
 }
 
@@ -273,7 +273,6 @@ impl PostgresExternalTableReader {
         config: ExternalTableConfig,
         rw_schema: Schema,
         pk_indices: Vec<usize>,
-        scan_limit: u32,
     ) -> ConnectorResult<Self> {
         tracing::info!(
             ?rw_schema,
@@ -334,32 +333,10 @@ impl PostgresExternalTableReader {
             .map(|f| Self::quote_column(&f.name))
             .join(",");
 
-        // prepare once
-        let prepared_scan_stmt = {
-            let primary_keys = pk_indices
-                .iter()
-                .map(|i| rw_schema.fields[*i].name.clone())
-                .collect_vec();
-
-            let table_name = SchemaTableName {
-                schema_name: config.schema.clone(),
-                table_name: config.table.clone(),
-            };
-            let order_key = Self::get_order_key(&primary_keys);
-            let scan_sql = format!(
-                "SELECT {} FROM {} WHERE {} ORDER BY {} LIMIT {scan_limit}",
-                field_names,
-                Self::get_normalized_table_name(&table_name),
-                Self::filter_expression(&primary_keys),
-                order_key,
-            );
-            client.prepare(&scan_sql).await?
-        };
-
         Ok(Self {
             rw_schema,
             field_names,
-            prepared_scan_stmt,
+            pk_indices,
             client: tokio::sync::Mutex::new(client),
         })
     }
@@ -385,7 +362,7 @@ impl PostgresExternalTableReader {
         table_name: SchemaTableName,
         start_pk_row: Option<OwnedRow>,
         primary_keys: Vec<String>,
-        limit: u32,
+        scan_limit: u32,
     ) {
         let order_key = Self::get_order_key(&primary_keys);
         let client = self.client.lock().await;
@@ -393,9 +370,29 @@ impl PostgresExternalTableReader {
 
         let stream = match start_pk_row {
             Some(ref pk_row) => {
+                // prepare the scan statement, since we may need to convert the RW data type to postgres data type
+                // e.g. varchar to uuid
+                let prepared_scan_stmt = {
+                    let primary_keys = self
+                        .pk_indices
+                        .iter()
+                        .map(|i| self.rw_schema.fields[*i].name.clone())
+                        .collect_vec();
+
+                    let order_key = Self::get_order_key(&primary_keys);
+                    let scan_sql = format!(
+                        "SELECT {} FROM {} WHERE {} ORDER BY {} LIMIT {scan_limit}",
+                        self.field_names,
+                        Self::get_normalized_table_name(&table_name),
+                        Self::filter_expression(&primary_keys),
+                        order_key,
+                    );
+                    client.prepare(&scan_sql).await?
+                };
+
                 let params: Vec<Option<ScalarAdapter>> = pk_row
                     .iter()
-                    .zip_eq_fast(self.prepared_scan_stmt.params())
+                    .zip_eq_fast(prepared_scan_stmt.params())
                     .map(|(datum, ty)| {
                         datum
                             .map(|scalar| ScalarAdapter::from_scalar(scalar, ty))
@@ -403,11 +400,11 @@ impl PostgresExternalTableReader {
                     })
                     .try_collect()?;
 
-                client.query_raw(&self.prepared_scan_stmt, &params).await?
+                client.query_raw(&prepared_scan_stmt, &params).await?
             }
             None => {
                 let sql = format!(
-                    "SELECT {} FROM {} ORDER BY {} LIMIT {limit}",
+                    "SELECT {} FROM {} ORDER BY {} LIMIT {scan_limit}",
                     self.field_names,
                     Self::get_normalized_table_name(&table_name),
                     order_key,
@@ -459,7 +456,6 @@ impl PostgresExternalTableReader {
 
 #[cfg(test)]
 mod tests {
-
     use std::collections::HashMap;
 
     use futures::pin_mut;
@@ -547,7 +543,7 @@ mod tests {
         let config =
             serde_json::from_value::<ExternalTableConfig>(serde_json::to_value(props).unwrap())
                 .unwrap();
-        let reader = PostgresExternalTableReader::new(config, rw_schema, vec![0, 1], 1000)
+        let reader = PostgresExternalTableReader::new(config, rw_schema, vec![0, 1])
             .await
             .unwrap();
 
diff --git a/src/connector/src/source/cdc/external/sql_server.rs b/src/connector/src/source/cdc/external/sql_server.rs
new file mode 100644
index 0000000000000..76b8e8e09fa41
--- /dev/null
+++ b/src/connector/src/source/cdc/external/sql_server.rs
@@ -0,0 +1,439 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::cmp::Ordering;
+
+use anyhow::Context;
+use futures::stream::BoxStream;
+use futures::{pin_mut, StreamExt, TryStreamExt};
+use futures_async_stream::try_stream;
+use itertools::Itertools;
+use risingwave_common::bail;
+use risingwave_common::catalog::{ColumnDesc, ColumnId, Schema};
+use risingwave_common::row::OwnedRow;
+use risingwave_common::types::{DataType, ScalarImpl};
+use serde_derive::{Deserialize, Serialize};
+use tiberius::{ColumnType, Config, Query, QueryItem};
+
+use crate::error::{ConnectorError, ConnectorResult};
+use crate::parser::{sql_server_row_to_owned_row, ScalarImplTiberiusWrapper};
+use crate::sink::sqlserver::SqlServerClient;
+use crate::source::cdc::external::{
+    CdcOffset, CdcOffsetParseFunc, DebeziumOffset, ExternalTableConfig, ExternalTableReader,
+    SchemaTableName,
+};
+
+// The maximum commit_lsn value in Sql Server
+const MAX_COMMIT_LSN: &str = "ffffffff:ffffffff:ffff";
+
+#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
+pub struct SqlServerOffset {
+    // https://learn.microsoft.com/en-us/answers/questions/1328359/how-to-accurately-sequence-change-data-capture-dat
+    pub change_lsn: String,
+    pub commit_lsn: String,
+}
+
+// only compare the lsn field
+impl PartialOrd for SqlServerOffset {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        match self.change_lsn.partial_cmp(&other.change_lsn) {
+            Some(Ordering::Equal) => self.commit_lsn.partial_cmp(&other.commit_lsn),
+            other => other,
+        }
+    }
+}
+
+impl SqlServerOffset {
+    pub fn parse_debezium_offset(offset: &str) -> ConnectorResult<Self> {
+        let dbz_offset: DebeziumOffset = serde_json::from_str(offset)
+            .with_context(|| format!("invalid upstream offset: {}", offset))?;
+
+        Ok(Self {
+            change_lsn: dbz_offset
+                .source_offset
+                .change_lsn
+                .context("invalid sql server change_lsn")?,
+            commit_lsn: dbz_offset
+                .source_offset
+                .commit_lsn
+                .context("invalid sql server commit_lsn")?,
+        })
+    }
+}
+
+pub struct SqlServerExternalTable {
+    column_descs: Vec<ColumnDesc>,
+    pk_names: Vec<String>,
+}
+
+impl SqlServerExternalTable {
+    pub async fn connect(config: ExternalTableConfig) -> ConnectorResult<Self> {
+        tracing::debug!("connect to sql server");
+
+        let mut client_config = Config::new();
+
+        client_config.host(&config.host);
+        client_config.database(&config.database);
+        client_config.port(config.port.parse::<u16>().unwrap());
+        client_config.authentication(tiberius::AuthMethod::sql_server(
+            &config.username,
+            &config.password,
+        ));
+        // TODO(kexiang): add ssl support
+        // TODO(kexiang): use trust_cert_ca, trust_cert is not secure
+        client_config.trust_cert();
+
+        let mut client = SqlServerClient::new_with_config(client_config).await?;
+
+        let mut column_descs = vec![];
+        let mut pk_names = vec![];
+        {
+            // With `WHERE 1 = 0`, we only fetch the metadata (column names and types) of the table
+            let sql = Query::new(format!(
+                "SELECT * FROM {} WHERE 1 = 0",
+                SqlServerExternalTableReader::get_normalized_table_name(&SchemaTableName {
+                    schema_name: config.schema.clone(),
+                    table_name: config.table.clone(),
+                }),
+            ));
+
+            let mut stream = sql.query(&mut client.inner_client).await?;
+            while let Some(item) = stream.try_next().await? {
+                match item {
+                    QueryItem::Metadata(meta) => {
+                        for col in meta.columns() {
+                            column_descs.push(ColumnDesc::named(
+                                col.name(),
+                                ColumnId::placeholder(),
+                                type_to_rw_type(&col.column_type())?,
+                            ));
+                        }
+                    }
+                    QueryItem::Row(row) => {
+                        unreachable!("Unexpected row: {:?}, `SELECT * FROM {} WHERE 1 = 0` should never return rows", row, config.table.clone());
+                    }
+                }
+            }
+        }
+        {
+            let sql = Query::new(format!(
+                "SELECT kcu.COLUMN_NAME
+                FROM
+                    INFORMATION_SCHEMA.TABLE_CONSTRAINTS AS tc
+                JOIN
+                    INFORMATION_SCHEMA.KEY_COLUMN_USAGE AS kcu
+                    ON tc.CONSTRAINT_NAME = kcu.CONSTRAINT_NAME AND
+                    tc.TABLE_SCHEMA = kcu.TABLE_SCHEMA AND
+                    tc.TABLE_NAME = kcu.TABLE_NAME
+                WHERE
+                    tc.CONSTRAINT_TYPE = 'PRIMARY KEY' AND
+                    tc.TABLE_SCHEMA = '{}' AND tc.TABLE_NAME = '{}'",
+                config.schema, config.table,
+            ));
+
+            let mut stream = sql.query(&mut client.inner_client).await?;
+            while let Some(item) = stream.try_next().await? {
+                match item {
+                    QueryItem::Metadata(_) => {}
+                    QueryItem::Row(row) => {
+                        let pk_name: &str = row.try_get(0)?.unwrap();
+                        pk_names.push(pk_name.to_string());
+                    }
+                }
+            }
+        }
+
+        Ok(Self {
+            column_descs,
+            pk_names,
+        })
+    }
+
+    pub fn column_descs(&self) -> &Vec<ColumnDesc> {
+        &self.column_descs
+    }
+
+    pub fn pk_names(&self) -> &Vec<String> {
+        &self.pk_names
+    }
+}
+
+fn type_to_rw_type(col_type: &ColumnType) -> ConnectorResult<DataType> {
+    let dtype = match col_type {
+        ColumnType::Bit => DataType::Boolean,
+        ColumnType::Bitn => DataType::Bytea,
+        ColumnType::Int1 => DataType::Int16,
+        ColumnType::Int2 => DataType::Int16,
+        ColumnType::Int4 => DataType::Int32,
+        ColumnType::Int8 => DataType::Int64,
+        ColumnType::Float4 => DataType::Float32,
+        ColumnType::Float8 => DataType::Float64,
+        ColumnType::Decimaln | ColumnType::Numericn => DataType::Decimal,
+        ColumnType::Daten => DataType::Date,
+        ColumnType::Timen => DataType::Time,
+        ColumnType::Datetime
+        | ColumnType::Datetimen
+        | ColumnType::Datetime2
+        | ColumnType::Datetime4 => DataType::Timestamp,
+        ColumnType::DatetimeOffsetn => DataType::Timestamptz,
+        ColumnType::NVarchar | ColumnType::NChar | ColumnType::NText | ColumnType::Text => {
+            DataType::Varchar
+        }
+        // Null, Guid, Image, Money, Money4, Intn, Bitn, Floatn, Xml, Udt, SSVariant, BigVarBin, BigVarChar, BigBinary, BigChar
+        mssql_type => {
+            // NOTES: user-defined enum type is classified as `Unknown`
+            tracing::warn!(
+                "Unknown Sql Server data type: {:?}, map to varchar",
+                mssql_type
+            );
+            DataType::Varchar
+        }
+    };
+    Ok(dtype)
+}
+
+#[derive(Debug)]
+pub struct SqlServerExternalTableReader {
+    rw_schema: Schema,
+    field_names: String,
+    client: tokio::sync::Mutex<SqlServerClient>,
+}
+
+impl ExternalTableReader for SqlServerExternalTableReader {
+    async fn current_cdc_offset(&self) -> ConnectorResult<CdcOffset> {
+        let mut client = self.client.lock().await;
+        // start a transaction to read max start_lsn.
+        let row = client
+            .inner_client
+            .simple_query(String::from("SELECT sys.fn_cdc_get_max_lsn()"))
+            .await?
+            .into_row()
+            .await?
+            .expect("No result returned by `SELECT sys.fn_cdc_get_max_lsn()`");
+        // An example of change_lsn or commit_lsn: "00000027:00000ac0:0002" from debezium
+        // sys.fn_cdc_get_max_lsn() returns a 10 bytes array, we convert it to a hex string here.
+        let max_lsn = match row.try_get::<&[u8], usize>(0)? {
+            Some(bytes) => {
+                let mut hex_string = String::with_capacity(bytes.len() * 2 + 2);
+                assert_eq!(
+                    bytes.len(),
+                    10,
+                    "sys.fn_cdc_get_max_lsn() should return a 10 bytes array."
+                );
+                for byte in &bytes[0..4] {
+                    hex_string.push_str(&format!("{:02x}", byte));
+                }
+                hex_string.push(':');
+                for byte in &bytes[4..8] {
+                    hex_string.push_str(&format!("{:02x}", byte));
+                }
+                hex_string.push(':');
+                for byte in &bytes[8..10] {
+                    hex_string.push_str(&format!("{:02x}", byte));
+                }
+                hex_string
+            }
+            None => bail!("None is returned by `SELECT sys.fn_cdc_get_max_lsn()`, please ensure Sql Server Agent is running."),
+        };
+
+        tracing::debug!("current max_lsn: {}", max_lsn);
+
+        Ok(CdcOffset::SqlServer(SqlServerOffset {
+            change_lsn: max_lsn,
+            commit_lsn: MAX_COMMIT_LSN.into(),
+        }))
+    }
+
+    fn snapshot_read(
+        &self,
+        table_name: SchemaTableName,
+        start_pk: Option<OwnedRow>,
+        primary_keys: Vec<String>,
+        limit: u32,
+    ) -> BoxStream<'_, ConnectorResult<OwnedRow>> {
+        self.snapshot_read_inner(table_name, start_pk, primary_keys, limit)
+    }
+}
+
+impl SqlServerExternalTableReader {
+    pub async fn new(
+        config: ExternalTableConfig,
+        rw_schema: Schema,
+        pk_indices: Vec<usize>,
+    ) -> ConnectorResult<Self> {
+        tracing::info!(
+            ?rw_schema,
+            ?pk_indices,
+            "create sql server external table reader"
+        );
+        let mut client_config = Config::new();
+
+        client_config.host(&config.host);
+        client_config.database(&config.database);
+        client_config.port(config.port.parse::<u16>().unwrap());
+        client_config.authentication(tiberius::AuthMethod::sql_server(
+            &config.username,
+            &config.password,
+        ));
+        // TODO(kexiang): add ssl support
+        // TODO(kexiang): use trust_cert_ca, trust_cert is not secure
+        client_config.trust_cert();
+
+        let client = SqlServerClient::new_with_config(client_config).await?;
+
+        let field_names = rw_schema
+            .fields
+            .iter()
+            .map(|f| Self::quote_column(&f.name))
+            .join(",");
+
+        Ok(Self {
+            rw_schema,
+            field_names,
+            client: tokio::sync::Mutex::new(client),
+        })
+    }
+
+    pub fn get_cdc_offset_parser() -> CdcOffsetParseFunc {
+        Box::new(move |offset| {
+            Ok(CdcOffset::SqlServer(
+                SqlServerOffset::parse_debezium_offset(offset)?,
+            ))
+        })
+    }
+
+    #[try_stream(boxed, ok = OwnedRow, error = ConnectorError)]
+    async fn snapshot_read_inner(
+        &self,
+        table_name: SchemaTableName,
+        start_pk_row: Option<OwnedRow>,
+        primary_keys: Vec<String>,
+        limit: u32,
+    ) {
+        let order_key = primary_keys
+            .iter()
+            .map(|col| Self::quote_column(col))
+            .join(",");
+        let mut sql = Query::new(if start_pk_row.is_none() {
+            format!(
+                "SELECT {} FROM {} ORDER BY {} OFFSET 0 ROWS FETCH NEXT {limit} ROWS ONLY",
+                self.field_names,
+                Self::get_normalized_table_name(&table_name),
+                order_key,
+            )
+        } else {
+            let filter_expr = Self::filter_expression(&primary_keys);
+            format!(
+                "SELECT {} FROM {} WHERE {} ORDER BY {} OFFSET 0 ROWS FETCH LIMIT {limit} ROWS ONLY",
+                self.field_names,
+                Self::get_normalized_table_name(&table_name),
+                filter_expr,
+                order_key,
+            )
+        });
+
+        let mut client = self.client.lock().await;
+
+        // FIXME(kexiang): Set session timezone to UTC
+        if let Some(pk_row) = start_pk_row {
+            let params: Vec<Option<ScalarImpl>> = pk_row.into_iter().collect();
+            for param in params {
+                // primary key should not be null, so it's safe to unwrap
+                sql.bind(ScalarImplTiberiusWrapper::from(param.unwrap()));
+            }
+        }
+
+        let stream = sql.query(&mut client.inner_client).await?.into_row_stream();
+
+        let row_stream = stream.map(|res| {
+            // convert sql server row into OwnedRow
+            let mut row = res?;
+            Ok::<_, ConnectorError>(sql_server_row_to_owned_row(&mut row, &self.rw_schema))
+        });
+
+        pin_mut!(row_stream);
+
+        #[for_await]
+        for row in row_stream {
+            let row = row?;
+            yield row;
+        }
+    }
+
+    pub fn get_normalized_table_name(table_name: &SchemaTableName) -> String {
+        format!(
+            "\"{}\".\"{}\"",
+            table_name.schema_name, table_name.table_name
+        )
+    }
+
+    // sql server cannot leverage the given key to narrow down the range of scan,
+    // we need to rewrite the comparison conditions by our own.
+    // (a, b) > (x, y) => ("a" > @P1) OR (("a" = @P1) AND ("b" > @P2))
+    fn filter_expression(columns: &[String]) -> String {
+        let mut conditions = vec![];
+        // push the first condition
+        conditions.push(format!("({} > @P{})", Self::quote_column(&columns[0]), 1));
+        for i in 2..=columns.len() {
+            // '=' condition
+            let mut condition = String::new();
+            for (j, col) in columns.iter().enumerate().take(i - 1) {
+                if j == 0 {
+                    condition.push_str(&format!("({} = @P{})", Self::quote_column(col), j + 1));
+                } else {
+                    condition.push_str(&format!(
+                        " AND ({} = @P{})",
+                        Self::quote_column(col),
+                        j + 1
+                    ));
+                }
+            }
+            // '>' condition
+            condition.push_str(&format!(
+                " AND ({} > @P{})",
+                Self::quote_column(&columns[i - 1]),
+                i
+            ));
+            conditions.push(format!("({})", condition));
+        }
+        if columns.len() > 1 {
+            conditions.join(" OR ")
+        } else {
+            conditions.join("")
+        }
+    }
+
+    fn quote_column(column: &str) -> String {
+        format!("\"{}\"", column)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::source::cdc::external::SqlServerExternalTableReader;
+
+    #[test]
+    fn test_sql_server_filter_expr() {
+        let cols = vec!["id".to_string()];
+        let expr = SqlServerExternalTableReader::filter_expression(&cols);
+        assert_eq!(expr, "(\"id\" > @P1)");
+
+        let cols = vec!["aa".to_string(), "bb".to_string(), "cc".to_string()];
+        let expr = SqlServerExternalTableReader::filter_expression(&cols);
+        assert_eq!(
+            expr,
+            "(\"aa\" > @P1) OR ((\"aa\" = @P1) AND (\"bb\" > @P2)) OR ((\"aa\" = @P1) AND (\"bb\" = @P2) AND (\"cc\" > @P3))"
+        );
+    }
+}
diff --git a/src/connector/src/source/cdc/mod.rs b/src/connector/src/source/cdc/mod.rs
index 2a4200b5cc4a0..605dd47ecd0e3 100644
--- a/src/connector/src/source/cdc/mod.rs
+++ b/src/connector/src/source/cdc/mod.rs
@@ -50,6 +50,7 @@ pub const MYSQL_CDC_CONNECTOR: &str = Mysql::CDC_CONNECTOR_NAME;
 pub const POSTGRES_CDC_CONNECTOR: &str = Postgres::CDC_CONNECTOR_NAME;
 pub const CITUS_CDC_CONNECTOR: &str = Citus::CDC_CONNECTOR_NAME;
 pub const MONGODB_CDC_CONNECTOR: &str = Mongodb::CDC_CONNECTOR_NAME;
+pub const SQL_SERVER_CDC_CONNECTOR: &str = SqlServer::CDC_CONNECTOR_NAME;
 
 pub trait CdcSourceTypeTrait: Send + Sync + Clone + 'static {
     const CDC_CONNECTOR_NAME: &'static str;
@@ -65,6 +66,7 @@ impl<'a> From<&'a str> for CdcSourceType {
             POSTGRES_CDC_CONNECTOR => CdcSourceType::Postgres,
             CITUS_CDC_CONNECTOR => CdcSourceType::Citus,
             MONGODB_CDC_CONNECTOR => CdcSourceType::Mongodb,
+            SQL_SERVER_CDC_CONNECTOR => CdcSourceType::SqlServer,
             _ => CdcSourceType::Unspecified,
         }
     }
@@ -77,6 +79,7 @@ impl CdcSourceType {
             CdcSourceType::Postgres => "Postgres",
             CdcSourceType::Citus => "Citus",
             CdcSourceType::Mongodb => "MongoDB",
+            CdcSourceType::SqlServer => "SQL Server",
             CdcSourceType::Unspecified => "Unspecified",
         }
     }
@@ -120,7 +123,7 @@ impl<T: CdcSourceTypeTrait> TryFromBTreeMap for CdcProperties<T> {
         properties: BTreeMap<String, String>,
         _deny_unknown_fields: bool,
     ) -> ConnectorResult<Self> {
-        let is_share_source = properties
+        let is_share_source: bool = properties
             .get(CDC_SHARING_MODE_KEY)
             .is_some_and(|v| v == "true");
         Ok(CdcProperties {
@@ -180,8 +183,6 @@ where
     }
 
     fn init_from_pb_cdc_table_desc(&mut self, table_desc: &ExternalTableDesc) {
-        let properties = table_desc.connect_properties.clone();
-
         let table_schema = TableSchema {
             columns: table_desc
                 .columns
@@ -197,7 +198,6 @@ where
             pk_indices: table_desc.stream_key.clone(),
         };
 
-        self.properties = properties;
         self.table_schema = table_schema;
         self.is_cdc_source_job = false;
         self.is_backfill_table = true;
diff --git a/src/connector/src/source/cdc/source/message.rs b/src/connector/src/source/cdc/source/message.rs
index f12d18339b527..831e242fe78de 100644
--- a/src/connector/src/source/cdc/source/message.rs
+++ b/src/connector/src/source/cdc/source/message.rs
@@ -13,11 +13,32 @@
 // limitations under the License.
 
 use risingwave_common::types::{DatumRef, ScalarRefImpl, Timestamptz};
-use risingwave_pb::connector_service::CdcMessage;
+use risingwave_pb::connector_service::{cdc_message, CdcMessage};
 
 use crate::source::base::SourceMessage;
 use crate::source::SourceMeta;
 
+#[derive(Clone, Debug)]
+pub enum CdcMessageType {
+    Unspecified,
+    Heartbeat,
+    Data,
+    TransactionMeta,
+    SchemaChange,
+}
+
+impl From<cdc_message::CdcMessageType> for CdcMessageType {
+    fn from(msg_type: cdc_message::CdcMessageType) -> Self {
+        match msg_type {
+            cdc_message::CdcMessageType::Data => CdcMessageType::Data,
+            cdc_message::CdcMessageType::Heartbeat => CdcMessageType::Heartbeat,
+            cdc_message::CdcMessageType::TransactionMeta => CdcMessageType::TransactionMeta,
+            cdc_message::CdcMessageType::SchemaChange => CdcMessageType::SchemaChange,
+            cdc_message::CdcMessageType::Unspecified => CdcMessageType::Unspecified,
+        }
+    }
+}
+
 #[derive(Debug, Clone)]
 pub struct DebeziumCdcMeta {
     db_name_prefix_len: usize,
@@ -25,11 +46,11 @@ pub struct DebeziumCdcMeta {
     pub full_table_name: String,
     // extracted from `payload.source.ts_ms`, the time that the change event was made in the database
     pub source_ts_ms: i64,
-    // Whether the message is a transaction metadata
-    pub is_transaction_meta: bool,
+    pub msg_type: CdcMessageType,
 }
 
 impl DebeziumCdcMeta {
+    // These `extract_xxx` methods are used to support the `INCLUDE TIMESTAMP/DATABASE_NAME/TABLE_NAME` feature
     pub fn extract_timestamp(&self) -> DatumRef<'_> {
         Some(ScalarRefImpl::Timestamptz(
             Timestamptz::from_millis(self.source_ts_ms).unwrap(),
@@ -48,20 +69,25 @@ impl DebeziumCdcMeta {
         ))
     }
 
-    pub fn new(full_table_name: String, source_ts_ms: i64, is_transaction_meta: bool) -> Self {
+    pub fn new(
+        full_table_name: String,
+        source_ts_ms: i64,
+        msg_type: cdc_message::CdcMessageType,
+    ) -> Self {
         // full_table_name is in the format of `database_name.table_name`
         let db_name_prefix_len = full_table_name.as_str().find('.').unwrap_or(0);
         Self {
             db_name_prefix_len,
             full_table_name,
             source_ts_ms,
-            is_transaction_meta,
+            msg_type: msg_type.into(),
         }
     }
 }
 
 impl From<CdcMessage> for SourceMessage {
     fn from(message: CdcMessage) -> Self {
+        let msg_type = message.get_msg_type().expect("invalid message type");
         SourceMessage {
             key: if message.key.is_empty() {
                 None // only data message has key
@@ -78,7 +104,7 @@ impl From<CdcMessage> for SourceMessage {
             meta: SourceMeta::DebeziumCdc(DebeziumCdcMeta::new(
                 message.full_table_name,
                 message.source_ts_ms,
-                message.is_transaction_meta,
+                msg_type,
             )),
         }
     }
diff --git a/src/connector/src/source/cdc/source/reader.rs b/src/connector/src/source/cdc/source/reader.rs
index 171b25bc94782..135daa5e04804 100644
--- a/src/connector/src/source/cdc/source/reader.rs
+++ b/src/connector/src/source/cdc/source/reader.rs
@@ -167,7 +167,10 @@ impl<T: CdcSourceTypeTrait> SplitReader for CdcSplitReader<T> {
         tracing::info!(?source_id, "cdc connector started");
 
         let instance = match T::source_type() {
-            CdcSourceType::Mysql | CdcSourceType::Postgres | CdcSourceType::Mongodb => Self {
+            CdcSourceType::Mysql
+            | CdcSourceType::Postgres
+            | CdcSourceType::Mongodb
+            | CdcSourceType::SqlServer => Self {
                 source_id: split.split_id() as u64,
                 start_offset: split.start_offset().clone(),
                 server_addr: None,
@@ -217,7 +220,7 @@ impl<T: CdcSourceTypeTrait> CdcSplitReader<T> {
                     tracing::trace!("receive {} cdc events ", events.len());
                     metrics
                         .connector_source_rows_received
-                        .with_label_values(&[source_type.as_str_name(), &source_id])
+                        .with_guarded_label_values(&[source_type.as_str_name(), &source_id])
                         .inc_by(events.len() as u64);
                     let msgs = events.into_iter().map(SourceMessage::from).collect_vec();
                     yield msgs;
diff --git a/src/connector/src/source/cdc/split.rs b/src/connector/src/source/cdc/split.rs
index d38200910f7ee..d6415135ea5ce 100644
--- a/src/connector/src/source/cdc/split.rs
+++ b/src/connector/src/source/cdc/split.rs
@@ -92,6 +92,11 @@ pub struct MongoDbCdcSplit {
     pub inner: CdcSplitBase,
 }
 
+#[derive(Clone, Serialize, Deserialize, Debug, PartialEq, Hash)]
+pub struct SqlServerCdcSplit {
+    pub inner: CdcSplitBase,
+}
+
 impl MySqlCdcSplit {
     pub fn new(split_id: u32, start_offset: Option<String>) -> Self {
         let split = CdcSplitBase {
@@ -214,6 +219,38 @@ impl CdcSplitTrait for MongoDbCdcSplit {
     }
 }
 
+impl SqlServerCdcSplit {
+    pub fn new(split_id: u32, start_offset: Option<String>) -> Self {
+        let split = CdcSplitBase {
+            split_id,
+            start_offset,
+            snapshot_done: false,
+        };
+        Self { inner: split }
+    }
+}
+
+impl CdcSplitTrait for SqlServerCdcSplit {
+    fn split_id(&self) -> u32 {
+        self.inner.split_id
+    }
+
+    fn start_offset(&self) -> &Option<String> {
+        &self.inner.start_offset
+    }
+
+    fn is_snapshot_done(&self) -> bool {
+        self.inner.snapshot_done
+    }
+
+    fn update_offset(&mut self, last_seen_offset: String) -> ConnectorResult<()> {
+        // if snapshot_done is already true, it will remain true
+        self.inner.snapshot_done = self.extract_snapshot_flag(last_seen_offset.as_str())?;
+        self.inner.start_offset = Some(last_seen_offset);
+        Ok(())
+    }
+}
+
 /// We use this struct to wrap the specific split, which act as an interface to other modules
 #[derive(Clone, Serialize, Deserialize, Debug, PartialEq, Hash)]
 pub struct DebeziumCdcSplit<T: CdcSourceTypeTrait> {
@@ -223,18 +260,19 @@ pub struct DebeziumCdcSplit<T: CdcSourceTypeTrait> {
     pub postgres_split: Option<PostgresCdcSplit>,
     pub citus_split: Option<PostgresCdcSplit>,
     pub mongodb_split: Option<MongoDbCdcSplit>,
+    pub sql_server_split: Option<SqlServerCdcSplit>,
 
     #[serde(skip)]
     pub _phantom: PhantomData<T>,
 }
 
 macro_rules! dispatch_cdc_split_inner {
-    ($dbz_split:expr, $as_type:tt, {$($cdc_source_type:tt),*}, $body:expr) => {
+    ($dbz_split:expr, $as_type:tt, {$({$cdc_source_type:tt, $cdc_source_split:tt}),*}, $body:expr) => {
         match T::source_type() {
             $(
                 CdcSourceType::$cdc_source_type => {
                     $crate::paste! {
-                        $dbz_split.[<$cdc_source_type:lower _split>]
+                        $dbz_split.[<$cdc_source_split>]
                             .[<as_ $as_type>]()
                             .expect(concat!(stringify!([<$cdc_source_type:lower>]), " split must exist"))
                             .$body
@@ -251,7 +289,13 @@ macro_rules! dispatch_cdc_split_inner {
 // call corresponding split method of the specific cdc source type
 macro_rules! dispatch_cdc_split {
     ($dbz_split:expr, $as_type:tt, $body:expr) => {
-      dispatch_cdc_split_inner!($dbz_split, $as_type, {Mysql, Postgres, Citus, Mongodb}, $body)
+        dispatch_cdc_split_inner!($dbz_split, $as_type, {
+            {Mysql, mysql_split},
+            {Postgres, postgres_split},
+            {Citus, citus_split},
+            {Mongodb, mongodb_split},
+            {SqlServer, sql_server_split}
+        }, $body)
     }
 }
 
@@ -280,6 +324,7 @@ impl<T: CdcSourceTypeTrait> DebeziumCdcSplit<T> {
             postgres_split: None,
             citus_split: None,
             mongodb_split: None,
+            sql_server_split: None,
             _phantom: PhantomData,
         };
         match T::source_type() {
@@ -299,6 +344,10 @@ impl<T: CdcSourceTypeTrait> DebeziumCdcSplit<T> {
                 let split = MongoDbCdcSplit::new(split_id, start_offset);
                 ret.mongodb_split = Some(split);
             }
+            CdcSourceType::SqlServer => {
+                let split = SqlServerCdcSplit::new(split_id, start_offset);
+                ret.sql_server_split = Some(split);
+            }
             CdcSourceType::Unspecified => {
                 unreachable!("invalid debezium split")
             }
diff --git a/src/connector/src/source/common.rs b/src/connector/src/source/common.rs
index abb80221e8fae..3acb85a87150e 100644
--- a/src/connector/src/source/common.rs
+++ b/src/connector/src/source/common.rs
@@ -49,7 +49,7 @@ pub(crate) async fn into_chunk_stream(
             for (split_id, msgs) in by_split_id {
                 metrics
                     .partition_input_count
-                    .with_label_values(&[
+                    .with_guarded_label_values(&[
                         &actor_id,
                         &source_id,
                         split_id,
@@ -65,7 +65,7 @@ pub(crate) async fn into_chunk_stream(
 
                 metrics
                     .partition_input_bytes
-                    .with_label_values(&[
+                    .with_guarded_label_values(&[
                         &actor_id,
                         &source_id,
                         split_id,
diff --git a/src/connector/src/source/datagen/source/reader.rs b/src/connector/src/source/datagen/source/reader.rs
index 992343058e9ef..e6c6db6af5a71 100644
--- a/src/connector/src/source/datagen/source/reader.rs
+++ b/src/connector/src/source/datagen/source/reader.rs
@@ -162,7 +162,7 @@ impl SplitReader for DatagenSplitReader {
                         .inspect_ok(move |stream_chunk| {
                             metrics
                                 .partition_input_count
-                                .with_label_values(&[
+                                .with_guarded_label_values(&[
                                     &actor_id,
                                     &source_id,
                                     &split_id,
diff --git a/src/connector/src/source/filesystem/opendal_source/mod.rs b/src/connector/src/source/filesystem/opendal_source/mod.rs
index 26a311f26eb8d..78c6ebf4cd8c7 100644
--- a/src/connector/src/source/filesystem/opendal_source/mod.rs
+++ b/src/connector/src/source/filesystem/opendal_source/mod.rs
@@ -26,7 +26,7 @@ pub mod opendal_reader;
 use self::opendal_enumerator::OpendalEnumerator;
 use self::opendal_reader::OpendalReader;
 use super::file_common::CompressionFormat;
-use super::s3::S3PropertiesCommon;
+pub use super::s3::S3PropertiesCommon;
 use super::OpendalFsSplit;
 use crate::error::ConnectorResult;
 use crate::source::{SourceProperties, UnknownFields};
diff --git a/src/connector/src/source/filesystem/opendal_source/opendal_reader.rs b/src/connector/src/source/filesystem/opendal_source/opendal_reader.rs
index a37a4999d90e5..5757452d2b4cd 100644
--- a/src/connector/src/source/filesystem/opendal_source/opendal_reader.rs
+++ b/src/connector/src/source/filesystem/opendal_source/opendal_reader.rs
@@ -20,20 +20,22 @@ use async_trait::async_trait;
 use futures::TryStreamExt;
 use futures_async_stream::try_stream;
 use opendal::Operator;
+use parquet::arrow::ParquetRecordBatchStreamBuilder;
 use risingwave_common::array::StreamChunk;
+use risingwave_common::util::tokio_util::compat::FuturesAsyncReadCompatExt;
 use tokio::io::{AsyncRead, BufReader};
 use tokio_util::io::{ReaderStream, StreamReader};
 
 use super::opendal_enumerator::OpendalEnumerator;
 use super::OpendalSource;
 use crate::error::ConnectorResult;
-use crate::parser::ParserConfig;
+use crate::parser::{ByteStreamSourceParserImpl, EncodingProperties, ParquetParser, ParserConfig};
 use crate::source::filesystem::file_common::CompressionFormat;
 use crate::source::filesystem::nd_streaming::need_nd_streaming;
 use crate::source::filesystem::{nd_streaming, OpendalFsSplit};
 use crate::source::{
-    into_chunk_stream, BoxChunkSourceStream, Column, SourceContextRef, SourceMessage, SourceMeta,
-    SplitMetaData, SplitReader,
+    BoxChunkSourceStream, Column, SourceContextRef, SourceMessage, SourceMeta, SplitMetaData,
+    SplitReader,
 };
 
 const STREAM_READER_CAPACITY: usize = 4096;
@@ -76,24 +78,54 @@ impl<Src: OpendalSource> OpendalReader<Src> {
     #[try_stream(boxed, ok = StreamChunk, error = crate::error::ConnectorError)]
     async fn into_stream_inner(self) {
         for split in self.splits {
-            let data_stream = Self::stream_read_object(
-                self.connector.op.clone(),
-                split,
-                self.source_ctx.clone(),
-                self.connector.compression_format.clone(),
-            );
-
-            let data_stream = if need_nd_streaming(&self.parser_config.specific.encoding_config) {
-                nd_streaming::split_stream(data_stream)
+            let source_ctx = self.source_ctx.clone();
+
+            let object_name = split.name.clone();
+
+            let msg_stream;
+
+            if let EncodingProperties::Parquet = &self.parser_config.specific.encoding_config {
+                // // If the format is "parquet", use `ParquetParser` to convert `record_batch` into stream chunk.
+                let reader: tokio_util::compat::Compat<opendal::FuturesAsyncReader> = self
+                    .connector
+                    .op
+                    .reader_with(&object_name)
+                    .into_future() // Unlike `rustc`, `try_stream` seems require manual `into_future`.
+                    .await?
+                    .into_futures_async_read(..)
+                    .await?
+                    .compat();
+                // For the Parquet format, we directly convert from a record batch to a stream chunk.
+                // Therefore, the offset of the Parquet file represents the current position in terms of the number of rows read from the file.
+                let record_batch_stream = ParquetRecordBatchStreamBuilder::new(reader)
+                    .await?
+                    .with_batch_size(self.source_ctx.source_ctrl_opts.chunk_size)
+                    .with_offset(split.offset)
+                    .build()?;
+
+                let parquet_parser = ParquetParser::new(
+                    self.parser_config.common.rw_columns.clone(),
+                    object_name,
+                    split.offset,
+                )?;
+                msg_stream = parquet_parser.into_stream(record_batch_stream);
             } else {
-                data_stream
-            };
-
-            let msg_stream = into_chunk_stream(
-                data_stream,
-                self.parser_config.clone(),
-                self.source_ctx.clone(),
-            );
+                let data_stream = Self::stream_read_object(
+                    self.connector.op.clone(),
+                    split,
+                    self.source_ctx.clone(),
+                    self.connector.compression_format.clone(),
+                );
+
+                let parser =
+                    ByteStreamSourceParserImpl::create(self.parser_config.clone(), source_ctx)
+                        .await?;
+                msg_stream = if need_nd_streaming(&self.parser_config.specific.encoding_config) {
+                    Box::pin(parser.into_stream(nd_streaming::split_stream(data_stream)))
+                } else {
+                    Box::pin(parser.into_stream(data_stream))
+                };
+            }
             #[for_await]
             for msg in msg_stream {
                 let msg = msg?;
@@ -115,15 +147,12 @@ impl<Src: OpendalSource> OpendalReader<Src> {
         let source_name = source_ctx.source_name.to_string();
         let max_chunk_size = source_ctx.source_ctrl_opts.chunk_size;
         let split_id = split.id();
-
         let object_name = split.name.clone();
-
         let reader = op
             .read_with(&object_name)
             .range(split.offset as u64..)
             .into_future() // Unlike `rustc`, `try_stream` seems require manual `into_future`.
             .await?;
-
         let stream_reader = StreamReader::new(
             reader.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e)),
         );
@@ -144,11 +173,10 @@ impl<Src: OpendalSource> OpendalReader<Src> {
             }
         };
 
-        let stream = ReaderStream::with_capacity(buf_reader, STREAM_READER_CAPACITY);
-
         let mut offset: usize = split.offset;
         let mut batch_size: usize = 0;
         let mut batch = Vec::new();
+        let stream = ReaderStream::with_capacity(buf_reader, STREAM_READER_CAPACITY);
         #[for_await]
         for read in stream {
             let bytes = read?;
@@ -168,7 +196,7 @@ impl<Src: OpendalSource> OpendalReader<Src> {
                 source_ctx
                     .metrics
                     .partition_input_bytes
-                    .with_label_values(&[
+                    .with_guarded_label_values(&[
                         &actor_id,
                         &source_id,
                         &split_id,
@@ -185,7 +213,13 @@ impl<Src: OpendalSource> OpendalReader<Src> {
             source_ctx
                 .metrics
                 .partition_input_bytes
-                .with_label_values(&[&actor_id, &source_id, &split_id, &source_name, &fragment_id])
+                .with_guarded_label_values(&[
+                    &actor_id,
+                    &source_id,
+                    &split_id,
+                    &source_name,
+                    &fragment_id,
+                ])
                 .inc_by(batch_size as u64);
             yield batch;
         }
diff --git a/src/connector/src/source/filesystem/s3/source/reader.rs b/src/connector/src/source/filesystem/s3/source/reader.rs
index 9f94dedbfacb5..7e02102686d00 100644
--- a/src/connector/src/source/filesystem/s3/source/reader.rs
+++ b/src/connector/src/source/filesystem/s3/source/reader.rs
@@ -124,7 +124,7 @@ impl S3FileReader {
                 source_ctx
                     .metrics
                     .partition_input_bytes
-                    .with_label_values(&[
+                    .with_guarded_label_values(&[
                         &actor_id,
                         &source_id,
                         &split_id,
@@ -141,7 +141,13 @@ impl S3FileReader {
             source_ctx
                 .metrics
                 .partition_input_bytes
-                .with_label_values(&[&actor_id, &source_id, &split_id, &source_name, &fragment_id])
+                .with_guarded_label_values(&[
+                    &actor_id,
+                    &source_id,
+                    &split_id,
+                    &source_name,
+                    &fragment_id,
+                ])
                 .inc_by(batch_size as u64);
             yield batch;
         }
diff --git a/src/connector/src/source/iceberg/mod.rs b/src/connector/src/source/iceberg/mod.rs
index f38f867e1925b..f101ff9ed6d4b 100644
--- a/src/connector/src/source/iceberg/mod.rs
+++ b/src/connector/src/source/iceberg/mod.rs
@@ -12,13 +12,19 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+pub mod parquet_file_reader;
+
 use std::collections::HashMap;
 
 use anyhow::anyhow;
 use async_trait::async_trait;
-use icelake::types::DataContentType;
+use futures_async_stream::for_await;
+use iceberg::scan::FileScanTask;
+use iceberg::spec::TableMetadata;
 use itertools::Itertools;
+pub use parquet_file_reader::*;
 use risingwave_common::bail;
+use risingwave_common::catalog::Schema;
 use risingwave_common::types::JsonbVal;
 use serde::{Deserialize, Serialize};
 
@@ -106,11 +112,38 @@ impl UnknownFields for IcebergProperties {
     }
 }
 
+#[derive(Debug, Clone, Eq, PartialEq, Hash, Serialize, Deserialize)]
+pub struct IcebergFileScanTaskJsonStr(String);
+
+impl IcebergFileScanTaskJsonStr {
+    pub fn deserialize(&self) -> FileScanTask {
+        serde_json::from_str(&self.0).unwrap()
+    }
+
+    pub fn serialize(task: &FileScanTask) -> Self {
+        Self(serde_json::to_string(task).unwrap())
+    }
+}
+
+#[derive(Debug, Clone, Eq, PartialEq, Hash, Serialize, Deserialize)]
+pub struct TableMetadataJsonStr(String);
+
+impl TableMetadataJsonStr {
+    pub fn deserialize(&self) -> TableMetadata {
+        serde_json::from_str(&self.0).unwrap()
+    }
+
+    pub fn serialize(metadata: &TableMetadata) -> Self {
+        Self(serde_json::to_string(metadata).unwrap())
+    }
+}
+
 #[derive(Debug, Clone, Eq, PartialEq, Hash, Serialize, Deserialize)]
 pub struct IcebergSplit {
     pub split_id: i64,
     pub snapshot_id: i64,
-    pub files: Vec<String>,
+    pub table_meta: TableMetadataJsonStr,
+    pub files: Vec<IcebergFileScanTaskJsonStr>,
 }
 
 impl SplitMetaData for IcebergSplit {
@@ -165,65 +198,64 @@ pub enum IcebergTimeTravelInfo {
 impl IcebergSplitEnumerator {
     pub async fn list_splits_batch(
         &self,
+        schema: Schema,
         time_traval_info: Option<IcebergTimeTravelInfo>,
         batch_parallelism: usize,
     ) -> ConnectorResult<Vec<IcebergSplit>> {
         if batch_parallelism == 0 {
             bail!("Batch parallelism is 0. Cannot split the iceberg files.");
         }
-        let table = self.config.load_table().await?;
+        let table = self.config.load_table_v2().await?;
         let snapshot_id = match time_traval_info {
             Some(IcebergTimeTravelInfo::Version(version)) => {
-                let Some(snapshot) = table.current_table_metadata().snapshot(version) else {
+                let Some(snapshot) = table.metadata().snapshot_by_id(version) else {
                     bail!("Cannot find the snapshot id in the iceberg table.");
                 };
-                snapshot.snapshot_id
+                snapshot.snapshot_id()
             }
             Some(IcebergTimeTravelInfo::TimestampMs(timestamp)) => {
-                match &table.current_table_metadata().snapshots {
-                    Some(snapshots) => {
-                        let snapshot = snapshots
-                            .iter()
-                            .filter(|snapshot| snapshot.timestamp_ms <= timestamp)
-                            .max_by_key(|snapshot| snapshot.timestamp_ms);
-                        match snapshot {
-                            Some(snapshot) => snapshot.snapshot_id,
-                            None => {
-                                // convert unix time to human readable time
-                                let time = chrono::DateTime::from_timestamp_millis(timestamp);
-                                if time.is_some() {
-                                    bail!("Cannot find a snapshot older than {}", time.unwrap());
-                                } else {
-                                    bail!("Cannot find a snapshot");
-                                }
-                            }
-                        }
-                    }
+                let snapshot = table
+                    .metadata()
+                    .snapshots()
+                    .filter(|snapshot| snapshot.timestamp().timestamp_millis() <= timestamp)
+                    .max_by_key(|snapshot| snapshot.timestamp().timestamp_millis());
+                match snapshot {
+                    Some(snapshot) => snapshot.snapshot_id(),
                     None => {
-                        bail!("Cannot find the snapshots in the iceberg table.");
+                        // convert unix time to human readable time
+                        let time = chrono::DateTime::from_timestamp_millis(timestamp);
+                        if time.is_some() {
+                            bail!("Cannot find a snapshot older than {}", time.unwrap());
+                        } else {
+                            bail!("Cannot find a snapshot");
+                        }
                     }
                 }
             }
-            None => match table.current_table_metadata().current_snapshot_id {
-                Some(snapshot_id) => snapshot_id,
+            None => match table.metadata().current_snapshot() {
+                Some(snapshot) => snapshot.snapshot_id(),
                 None => bail!("Cannot find the current snapshot id in the iceberg table."),
             },
         };
         let mut files = vec![];
-        for file in table
-            .data_files_of_snapshot(
-                table
-                    .current_table_metadata()
-                    .snapshot(snapshot_id)
-                    .expect("snapshot must exists"),
-            )
-            .await?
-        {
-            if file.content != DataContentType::Data {
-                bail!("Reading iceberg table with delete files is unsupported. Please try to compact the table first.");
-            }
-            files.push(file.file_path);
+
+        let scan = table
+            .scan()
+            .snapshot_id(snapshot_id)
+            .select(schema.names())
+            .build()
+            .map_err(|e| anyhow!(e))?;
+
+        let file_scan_stream = scan.plan_files().await.map_err(|e| anyhow!(e))?;
+
+        #[for_await]
+        for task in file_scan_stream {
+            let task = task.map_err(|e| anyhow!(e))?;
+            files.push(IcebergFileScanTaskJsonStr::serialize(&task));
         }
+
+        let table_meta = TableMetadataJsonStr::serialize(table.metadata());
+
         let split_num = batch_parallelism;
         // evenly split the files into splits based on the parallelism.
         let split_size = files.len() / split_num;
@@ -235,6 +267,7 @@ impl IcebergSplitEnumerator {
             let split = IcebergSplit {
                 split_id: i as i64,
                 snapshot_id,
+                table_meta: table_meta.clone(),
                 files: files[start..end].to_vec(),
             };
             splits.push(split);
diff --git a/src/connector/src/source/iceberg/parquet_file_reader.rs b/src/connector/src/source/iceberg/parquet_file_reader.rs
new file mode 100644
index 0000000000000..6e323f4aec9ba
--- /dev/null
+++ b/src/connector/src/source/iceberg/parquet_file_reader.rs
@@ -0,0 +1,132 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::HashMap;
+use std::ops::Range;
+use std::sync::Arc;
+
+use anyhow::anyhow;
+use bytes::Bytes;
+use futures::future::BoxFuture;
+use futures::TryFutureExt;
+use iceberg::io::{
+    FileIOBuilder, FileMetadata, FileRead, S3_ACCESS_KEY_ID, S3_REGION, S3_SECRET_ACCESS_KEY,
+};
+use iceberg::{Error, ErrorKind};
+use opendal::layers::RetryLayer;
+use opendal::services::S3;
+use opendal::Operator;
+use parquet::arrow::async_reader::{AsyncFileReader, MetadataLoader};
+use parquet::arrow::ParquetRecordBatchStreamBuilder;
+use parquet::file::metadata::ParquetMetaData;
+use url::Url;
+
+pub struct ParquetFileReader<R: FileRead> {
+    meta: FileMetadata,
+    r: R,
+}
+
+impl<R: FileRead> ParquetFileReader<R> {
+    pub fn new(meta: FileMetadata, r: R) -> Self {
+        Self { meta, r }
+    }
+}
+
+impl<R: FileRead> AsyncFileReader for ParquetFileReader<R> {
+    fn get_bytes(&mut self, range: Range<usize>) -> BoxFuture<'_, parquet::errors::Result<Bytes>> {
+        Box::pin(
+            self.r
+                .read(range.start as _..range.end as _)
+                .map_err(|err| parquet::errors::ParquetError::External(Box::new(err))),
+        )
+    }
+
+    fn get_metadata(&mut self) -> BoxFuture<'_, parquet::errors::Result<Arc<ParquetMetaData>>> {
+        Box::pin(async move {
+            let file_size = self.meta.size;
+            let mut loader = MetadataLoader::load(self, file_size as usize, None).await?;
+            loader.load_page_index(false, false).await?;
+            Ok(Arc::new(loader.finish()))
+        })
+    }
+}
+
+pub async fn create_parquet_stream_builder(
+    s3_region: String,
+    s3_access_key: String,
+    s3_secret_key: String,
+    location: String,
+) -> Result<ParquetRecordBatchStreamBuilder<ParquetFileReader<impl FileRead>>, anyhow::Error> {
+    let mut props = HashMap::new();
+    props.insert(S3_REGION, s3_region.clone());
+    props.insert(S3_ACCESS_KEY_ID, s3_access_key.clone());
+    props.insert(S3_SECRET_ACCESS_KEY, s3_secret_key.clone());
+
+    let file_io_builder = FileIOBuilder::new("s3");
+    let file_io = file_io_builder
+        .with_props(props.into_iter())
+        .build()
+        .map_err(|e| anyhow!(e))?;
+    let parquet_file = file_io.new_input(&location).map_err(|e| anyhow!(e))?;
+
+    let parquet_metadata = parquet_file.metadata().await.map_err(|e| anyhow!(e))?;
+    let parquet_reader = parquet_file.reader().await.map_err(|e| anyhow!(e))?;
+    let parquet_file_reader = ParquetFileReader::new(parquet_metadata, parquet_reader);
+
+    ParquetRecordBatchStreamBuilder::new(parquet_file_reader)
+        .await
+        .map_err(|e| anyhow!(e))
+}
+
+pub async fn list_s3_directory(
+    s3_region: String,
+    s3_access_key: String,
+    s3_secret_key: String,
+    dir: String,
+) -> Result<Vec<String>, anyhow::Error> {
+    let url = Url::parse(&dir)?;
+    let bucket = url.host_str().ok_or_else(|| {
+        Error::new(
+            ErrorKind::DataInvalid,
+            format!("Invalid s3 url: {}, missing bucket", dir),
+        )
+    })?;
+
+    let prefix = format!("s3://{}/", bucket);
+    if dir.starts_with(&prefix) {
+        let mut builder = S3::default();
+        builder
+            .region(&s3_region)
+            .access_key_id(&s3_access_key)
+            .secret_access_key(&s3_secret_key)
+            .bucket(bucket);
+        let op = Operator::new(builder)?
+            .layer(RetryLayer::default())
+            .finish();
+
+        op.list(&dir[prefix.len()..])
+            .await
+            .map_err(|e| anyhow!(e))
+            .map(|list| {
+                list.into_iter()
+                    .map(|entry| prefix.to_string() + entry.path())
+                    .collect()
+            })
+    } else {
+        Err(Error::new(
+            ErrorKind::DataInvalid,
+            format!("Invalid s3 url: {}, should start with {}", dir, prefix),
+        ))?
+    }
+}
diff --git a/src/connector/src/source/kafka/enumerator/client.rs b/src/connector/src/source/kafka/enumerator/client.rs
index d4dddad67bfdf..ff007076c1338 100644
--- a/src/connector/src/source/kafka/enumerator/client.rs
+++ b/src/connector/src/source/kafka/enumerator/client.rs
@@ -82,7 +82,7 @@ impl SplitEnumerator for KafkaSplitEnumerator {
             Some("latest") => KafkaEnumeratorOffset::Latest,
             None => KafkaEnumeratorOffset::Earliest,
             _ => bail!(
-                "properties `scan_startup_mode` only support earliest and latest or leave it empty"
+                "properties `scan_startup_mode` only supports earliest and latest or leaving it empty"
             ),
         };
 
@@ -361,7 +361,7 @@ impl KafkaSplitEnumerator {
         self.context
             .metrics
             .high_watermark
-            .with_label_values(&[
+            .with_guarded_label_values(&[
                 &self.context.info.source_id.to_string(),
                 &partition.to_string(),
             ])
diff --git a/src/connector/src/source/kafka/source/reader.rs b/src/connector/src/source/kafka/source/reader.rs
index c54b0ac27655a..798069ddb12f9 100644
--- a/src/connector/src/source/kafka/source/reader.rs
+++ b/src/connector/src/source/kafka/source/reader.rs
@@ -158,7 +158,7 @@ impl KafkaSplitReader {
         self.source_ctx
             .metrics
             .latest_message_id
-            .with_label_values(&[
+            .with_guarded_label_values(&[
                 // source name is not available here
                 &self.source_ctx.source_id.to_string(),
                 &self.source_ctx.actor_id.to_string(),
diff --git a/src/connector/src/source/kinesis/enumerator/client.rs b/src/connector/src/source/kinesis/enumerator/client.rs
index 840def08f6855..423516fa5bd47 100644
--- a/src/connector/src/source/kinesis/enumerator/client.rs
+++ b/src/connector/src/source/kinesis/enumerator/client.rs
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use anyhow::Context as _;
+use anyhow::anyhow;
 use async_trait::async_trait;
 use aws_sdk_kinesis::types::Shard;
 use aws_sdk_kinesis::Client as kinesis_client;
@@ -52,14 +52,26 @@ impl SplitEnumerator for KinesisSplitEnumerator {
         let mut shard_collect: Vec<Shard> = Vec::new();
 
         loop {
-            let list_shard_output = self
-                .client
-                .list_shards()
-                .set_next_token(next_token)
-                .stream_name(&self.stream_name)
-                .send()
-                .await
-                .context("failed to list kinesis shards")?;
+            let mut req = self.client.list_shards();
+            if let Some(token) = next_token.take() {
+                req = req.next_token(token);
+            } else {
+                req = req.stream_name(&self.stream_name);
+            }
+
+            let list_shard_output = match req.send().await {
+                Ok(output) => output,
+                Err(e) => {
+                    if let Some(e_inner) = e.as_service_error()
+                        && e_inner.is_expired_next_token_exception()
+                    {
+                        tracing::info!("Kinesis ListShard token expired, retrying...");
+                        next_token = None;
+                        continue;
+                    }
+                    return Err(anyhow!(e).context("failed to list kinesis shards").into());
+                }
+            };
             match list_shard_output.shards {
                 Some(shard) => shard_collect.extend(shard),
                 None => bail!("no shards in stream {}", &self.stream_name),
diff --git a/src/connector/src/source/kinesis/source/reader.rs b/src/connector/src/source/kinesis/source/reader.rs
index b728270dbd821..4bac1fb1e49d2 100644
--- a/src/connector/src/source/kinesis/source/reader.rs
+++ b/src/connector/src/source/kinesis/source/reader.rs
@@ -91,7 +91,7 @@ impl SplitReader for KinesisSplitReader {
         if !matches!(start_position, KinesisOffset::Timestamp(_))
             && properties.timestamp_offset.is_some()
         {
-            bail!("scan.startup.mode need to be set to 'timestamp' if you want to start with a specific timestamp");
+            bail!("scan.startup.mode needs to be set to 'timestamp' if you want to start with a specific timestamp");
         }
 
         let stream_name = properties.common.stream_name.clone();
@@ -123,6 +123,7 @@ impl KinesisSplitReader {
     #[try_stream(ok = Vec < SourceMessage >, error = crate::error::ConnectorError)]
     async fn into_data_stream(mut self) {
         self.new_shard_iter().await?;
+        let mut finish_flag = false;
         loop {
             if self.shard_iter.is_none() {
                 tracing::warn!(
@@ -135,6 +136,18 @@ impl KinesisSplitReader {
             }
             match self.get_records().await {
                 Ok(resp) => {
+                    if resp.millis_behind_latest.is_none()
+                        && let Some(shard) = &resp.child_shards
+                        && !shard.is_empty()
+                    {
+                        // according to the doc https://docs.rs/aws-sdk-kinesis/latest/aws_sdk_kinesis/operation/get_records/struct.GetRecordsOutput.html
+                        //
+                        // > The list of the current shard's child shards, returned in the GetRecords API's response only when the end of the current shard is reached.
+                        //
+                        // It means the current shard is finished, ie. inactive, and we should stop reading it. Checking `millis_behind_latest` is a double check.
+                        // Other executors are going to read the child shards.
+                        finish_flag = true;
+                    }
                     self.shard_iter = resp.next_shard_iterator().map(String::from);
                     let chunk = (resp.records().iter())
                         .map(|r| from_kinesis_record(r, self.split_id.clone()))
@@ -150,6 +163,13 @@ impl KinesisSplitReader {
                         self.latest_offset
                     );
                     yield chunk;
+                    if finish_flag {
+                        tracing::info!(
+                            "shard {:?} reaches the end and is inactive, stop reading",
+                            self.shard_id
+                        );
+                        break;
+                    }
                 }
                 Err(SdkError::ServiceError(e)) if e.err().is_resource_not_found_exception() => {
                     tracing::warn!("shard {:?} is closed, stop reading", self.shard_id);
@@ -199,7 +219,7 @@ impl KinesisSplitReader {
                 }
                 Err(e) => {
                     let error = anyhow!(e).context(format!(
-                        "Kinesis got a unhandled error on stream {:?}, shard {:?}",
+                        "Kinesis got an unhandled error on stream {:?}, shard {:?}",
                         self.stream_name, self.shard_id
                     ));
                     tracing::error!(error = %error.as_report());
diff --git a/src/connector/src/source/monitor/metrics.rs b/src/connector/src/source/monitor/metrics.rs
index bf43ca44212ae..88a2e57ec4ec6 100644
--- a/src/connector/src/source/monitor/metrics.rs
+++ b/src/connector/src/source/monitor/metrics.rs
@@ -14,19 +14,21 @@
 
 use std::sync::{Arc, LazyLock};
 
-use prometheus::core::{AtomicI64, AtomicU64, GenericCounterVec, GenericGaugeVec};
-use prometheus::{
-    exponential_buckets, histogram_opts, register_histogram_vec_with_registry,
-    register_int_counter_vec_with_registry, register_int_gauge_vec_with_registry, HistogramVec,
-    Registry,
+use prometheus::{exponential_buckets, histogram_opts, Registry};
+use risingwave_common::metrics::{
+    LabelGuardedHistogramVec, LabelGuardedIntCounterVec, LabelGuardedIntGaugeVec,
 };
 use risingwave_common::monitor::GLOBAL_METRICS_REGISTRY;
+use risingwave_common::{
+    register_guarded_histogram_vec_with_registry, register_guarded_int_counter_vec_with_registry,
+    register_guarded_int_gauge_vec_with_registry,
+};
 
 use crate::source::kafka::stats::RdKafkaStats;
 
 #[derive(Debug, Clone)]
 pub struct EnumeratorMetrics {
-    pub high_watermark: GenericGaugeVec<AtomicI64>,
+    pub high_watermark: LabelGuardedIntGaugeVec<2>,
 }
 
 pub static GLOBAL_ENUMERATOR_METRICS: LazyLock<EnumeratorMetrics> =
@@ -34,7 +36,7 @@ pub static GLOBAL_ENUMERATOR_METRICS: LazyLock<EnumeratorMetrics> =
 
 impl EnumeratorMetrics {
     fn new(registry: &Registry) -> Self {
-        let high_watermark = register_int_gauge_vec_with_registry!(
+        let high_watermark = register_guarded_int_gauge_vec_with_registry!(
             "source_kafka_high_watermark",
             "High watermark for a exec per partition",
             &["source_id", "partition"],
@@ -57,18 +59,18 @@ impl Default for EnumeratorMetrics {
 
 #[derive(Debug, Clone)]
 pub struct SourceMetrics {
-    pub partition_input_count: GenericCounterVec<AtomicU64>,
+    pub partition_input_count: LabelGuardedIntCounterVec<5>,
 
     // **Note**: for normal messages, the metric is the message's payload size.
     // For messages from load generator, the metric is the size of stream chunk.
-    pub partition_input_bytes: GenericCounterVec<AtomicU64>,
+    pub partition_input_bytes: LabelGuardedIntCounterVec<5>,
     /// Report latest message id
-    pub latest_message_id: GenericGaugeVec<AtomicI64>,
+    pub latest_message_id: LabelGuardedIntGaugeVec<3>,
     pub rdkafka_native_metric: Arc<RdKafkaStats>,
 
-    pub connector_source_rows_received: GenericCounterVec<AtomicU64>,
+    pub connector_source_rows_received: LabelGuardedIntCounterVec<2>,
 
-    pub direct_cdc_event_lag_latency: HistogramVec,
+    pub direct_cdc_event_lag_latency: LabelGuardedHistogramVec<1>,
 }
 
 pub static GLOBAL_SOURCE_METRICS: LazyLock<SourceMetrics> =
@@ -76,7 +78,7 @@ pub static GLOBAL_SOURCE_METRICS: LazyLock<SourceMetrics> =
 
 impl SourceMetrics {
     fn new(registry: &Registry) -> Self {
-        let partition_input_count = register_int_counter_vec_with_registry!(
+        let partition_input_count = register_guarded_int_counter_vec_with_registry!(
             "source_partition_input_count",
             "Total number of rows that have been input from specific partition",
             &[
@@ -89,7 +91,7 @@ impl SourceMetrics {
             registry
         )
         .unwrap();
-        let partition_input_bytes = register_int_counter_vec_with_registry!(
+        let partition_input_bytes = register_guarded_int_counter_vec_with_registry!(
             "source_partition_input_bytes",
             "Total bytes that have been input from specific partition",
             &[
@@ -102,7 +104,7 @@ impl SourceMetrics {
             registry
         )
         .unwrap();
-        let latest_message_id = register_int_gauge_vec_with_registry!(
+        let latest_message_id = register_guarded_int_gauge_vec_with_registry!(
             "source_latest_message_id",
             "Latest message id for a exec per partition",
             &["source_id", "actor_id", "partition"],
@@ -110,7 +112,7 @@ impl SourceMetrics {
         )
         .unwrap();
 
-        let connector_source_rows_received = register_int_counter_vec_with_registry!(
+        let connector_source_rows_received = register_guarded_int_counter_vec_with_registry!(
             "source_rows_received",
             "Number of rows received by source",
             &["source_type", "source_id"],
@@ -124,7 +126,7 @@ impl SourceMetrics {
             exponential_buckets(1.0, 2.0, 21).unwrap(), // max 1048s
         );
         let direct_cdc_event_lag_latency =
-            register_histogram_vec_with_registry!(opts, &["table_name"], registry).unwrap();
+            register_guarded_histogram_vec_with_registry!(opts, &["table_name"], registry).unwrap();
 
         let rdkafka_native_metric = Arc::new(RdKafkaStats::new(registry.clone()));
         SourceMetrics {
diff --git a/src/connector/src/source/nexmark/source/reader.rs b/src/connector/src/source/nexmark/source/reader.rs
index aee7c178d116a..ebcbc0b0aaf32 100644
--- a/src/connector/src/source/nexmark/source/reader.rs
+++ b/src/connector/src/source/nexmark/source/reader.rs
@@ -122,7 +122,7 @@ impl SplitReader for NexmarkSplitReader {
                 .inspect_ok(move |chunk: &StreamChunk| {
                     metrics
                         .partition_input_count
-                        .with_label_values(&[
+                        .with_guarded_label_values(&[
                             &actor_id,
                             &source_id,
                             &split_id,
@@ -132,7 +132,7 @@ impl SplitReader for NexmarkSplitReader {
                         .inc_by(chunk.cardinality() as u64);
                     metrics
                         .partition_input_bytes
-                        .with_label_values(&[
+                        .with_guarded_label_values(&[
                             &actor_id,
                             &source_id,
                             &split_id,
diff --git a/src/connector/src/source/pulsar/enumerator/client.rs b/src/connector/src/source/pulsar/enumerator/client.rs
index dddcf927d0c3f..ef79a8f576d3e 100644
--- a/src/connector/src/source/pulsar/enumerator/client.rs
+++ b/src/connector/src/source/pulsar/enumerator/client.rs
@@ -65,7 +65,7 @@ impl SplitEnumerator for PulsarSplitEnumerator {
             None => PulsarEnumeratorOffset::Earliest,
             _ => {
                 bail!(
-                    "properties `startup_mode` only support earliest and latest or leave it empty"
+                    "properties `startup_mode` only supports earliest and latest or leaving it empty"
                 );
             }
         };
diff --git a/src/connector/src/source/reader/desc.rs b/src/connector/src/source/reader/desc.rs
index 44d2effe51606..af607d2537ea6 100644
--- a/src/connector/src/source/reader/desc.rs
+++ b/src/connector/src/source/reader/desc.rs
@@ -12,7 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::BTreeMap;
 use std::sync::Arc;
 
 use risingwave_common::bail;
@@ -29,6 +28,7 @@ use crate::parser::additional_columns::source_add_partition_offset_cols;
 use crate::parser::{EncodingProperties, ProtocolProperties, SpecificParserConfig};
 use crate::source::monitor::SourceMetrics;
 use crate::source::{SourceColumnDesc, SourceColumnType, UPSTREAM_SOURCE_KEY};
+use crate::WithOptionsSecResolved;
 
 pub const DEFAULT_CONNECTOR_MESSAGE_BUFFER_SIZE: usize = 16;
 
@@ -55,7 +55,7 @@ pub struct SourceDescBuilder {
     columns: Vec<ColumnCatalog>,
     metrics: Arc<SourceMetrics>,
     row_id_index: Option<usize>,
-    with_properties: BTreeMap<String, String>,
+    with_properties: WithOptionsSecResolved,
     source_info: PbStreamSourceInfo,
     connector_message_buffer_size: usize,
     pk_indices: Vec<usize>,
@@ -66,7 +66,7 @@ impl SourceDescBuilder {
         columns: Vec<PbColumnCatalog>,
         metrics: Arc<SourceMetrics>,
         row_id_index: Option<usize>,
-        with_properties: BTreeMap<String, String>,
+        with_properties: WithOptionsSecResolved,
         source_info: PbStreamSourceInfo,
         connector_message_buffer_size: usize,
         pk_indices: Vec<usize>,
@@ -199,11 +199,13 @@ pub mod test_utils {
                 ))
             })
             .collect();
+        let options_with_secret =
+            crate::WithOptionsSecResolved::without_secrets(with_properties.clone());
         SourceDescBuilder {
             columns,
             metrics: Default::default(),
             row_id_index,
-            with_properties,
+            with_properties: options_with_secret,
             source_info,
             connector_message_buffer_size: DEFAULT_CONNECTOR_MESSAGE_BUFFER_SIZE,
             pk_indices,
diff --git a/src/connector/src/source/reader/fs_reader.rs b/src/connector/src/source/reader/fs_reader.rs
index 5bd139e70983d..ae05bc64ca1a5 100644
--- a/src/connector/src/source/reader/fs_reader.rs
+++ b/src/connector/src/source/reader/fs_reader.rs
@@ -14,7 +14,6 @@
 
 #![deprecated = "will be replaced by new fs source (list + fetch)"]
 
-use std::collections::BTreeMap;
 use std::sync::Arc;
 
 use anyhow::Context;
@@ -22,26 +21,26 @@ use futures::stream::pending;
 use futures::StreamExt;
 use risingwave_common::catalog::ColumnId;
 
-use crate::dispatch_source_prop;
 use crate::error::ConnectorResult;
 use crate::parser::{CommonParserConfig, ParserConfig, SpecificParserConfig};
 use crate::source::{
     create_split_reader, BoxChunkSourceStream, ConnectorProperties, ConnectorState,
     SourceColumnDesc, SourceContext, SplitReader,
 };
+use crate::{dispatch_source_prop, WithOptionsSecResolved};
 
 #[derive(Clone, Debug)]
 pub struct FsSourceReader {
     pub config: ConnectorProperties,
     pub columns: Vec<SourceColumnDesc>,
-    pub properties: BTreeMap<String, String>,
+    pub properties: WithOptionsSecResolved,
     pub parser_config: SpecificParserConfig,
 }
 
 impl FsSourceReader {
     #[allow(clippy::too_many_arguments)]
     pub fn new(
-        properties: BTreeMap<String, String>,
+        properties: WithOptionsSecResolved,
         columns: Vec<SourceColumnDesc>,
         parser_config: SpecificParserConfig,
     ) -> ConnectorResult<Self> {
diff --git a/src/connector/src/source/reader/reader.rs b/src/connector/src/source/reader/reader.rs
index 02012841c5a48..10d549df0c49a 100644
--- a/src/connector/src/source/reader/reader.rs
+++ b/src/connector/src/source/reader/reader.rs
@@ -12,7 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::BTreeMap;
 use std::sync::Arc;
 
 use anyhow::Context;
@@ -26,7 +25,6 @@ use risingwave_common::catalog::ColumnId;
 use rw_futures_util::select_all;
 use thiserror_ext::AsReport as _;
 
-use crate::dispatch_source_prop;
 use crate::error::ConnectorResult;
 use crate::parser::{CommonParserConfig, ParserConfig, SpecificParserConfig};
 use crate::source::filesystem::opendal_source::opendal_enumerator::OpendalEnumerator;
@@ -38,6 +36,7 @@ use crate::source::{
     create_split_reader, BoxChunkSourceStream, BoxTryStream, Column, ConnectorProperties,
     ConnectorState, SourceColumnDesc, SourceContext, SplitReader, WaitCheckpointTask,
 };
+use crate::{dispatch_source_prop, WithOptionsSecResolved};
 
 #[derive(Clone, Debug)]
 pub struct SourceReader {
@@ -49,7 +48,7 @@ pub struct SourceReader {
 
 impl SourceReader {
     pub fn new(
-        properties: BTreeMap<String, String>,
+        properties: WithOptionsSecResolved,
         columns: Vec<SourceColumnDesc>,
         connector_message_buffer_size: usize,
         parser_config: SpecificParserConfig,
@@ -108,9 +107,7 @@ impl SourceReader {
     /// Refer to `WaitCheckpointWorker` for more details.
     pub async fn create_wait_checkpoint_task(&self) -> ConnectorResult<Option<WaitCheckpointTask>> {
         Ok(match &self.config {
-            ConnectorProperties::PostgresCdc(_prop) => {
-                Some(WaitCheckpointTask::CommitCdcOffset(None))
-            }
+            ConnectorProperties::PostgresCdc(_) => Some(WaitCheckpointTask::CommitCdcOffset(None)),
             ConnectorProperties::GooglePubsub(prop) => Some(WaitCheckpointTask::AckPubsubMessage(
                 prop.subscription_client().await?,
                 vec![],
diff --git a/src/connector/src/test_data/union-schema.avsc b/src/connector/src/test_data/union-schema.avsc
deleted file mode 100644
index 4176d8f771f09..0000000000000
--- a/src/connector/src/test_data/union-schema.avsc
+++ /dev/null
@@ -1,88 +0,0 @@
-{
-  "name": "test_student",
-  "type": "record",
-  "fields": [
-    {
-      "name": "id",
-      "type": "int",
-      "default": 0
-    },
-    {
-      "name": "age",
-      "type": ["null", "int"]
-    },
-    {
-      "name": "sequence_id",
-      "type": ["null", "long"]
-    },
-    {
-      "name": "name",
-      "type": ["null", "string"],
-      "default": null
-    },
-    {
-      "name": "score",
-      "type": [ "float", "null" ],
-      "default": 1.0
-    },
-    {
-      "name": "avg_score",
-      "type": ["null", "double"]
-    },
-    {
-      "name": "is_lasted",
-      "type": ["null", "boolean"]
-    },
-    {
-      "name": "entrance_date",
-      "type": [
-        "null",
-        {
-          "type": "int",
-          "logicalType": "date",
-          "arg.properties": {
-            "range": {
-              "min": 1,
-              "max": 19374
-            }
-          }
-        }
-      ],
-      "default": null
-    },
-    {
-      "name": "birthday",
-      "type": [
-        "null",
-          {
-            "type": "long",
-            "logicalType": "timestamp-millis",
-            "arg.properties": {
-              "range": {
-                "min": 1,
-                "max": 1673970376213
-              }
-            }
-          }
-      ],
-      "default": null
-    },
-    {
-      "name": "anniversary",
-      "type": [
-        "null",
-        {
-          "type" : "long",
-          "logicalType": "timestamp-micros",
-          "arg.properties": {
-            "range": {
-              "min": 1,
-              "max": 1673970376213000
-            }
-          }
-        }
-      ],
-      "default": null
-    }
-  ]
-}
\ No newline at end of file
diff --git a/src/connector/src/with_options.rs b/src/connector/src/with_options.rs
index 154586d770522..eef5ccbd9cbfa 100644
--- a/src/connector/src/with_options.rs
+++ b/src/connector/src/with_options.rs
@@ -14,6 +14,9 @@
 
 use std::collections::{BTreeMap, HashMap};
 
+use risingwave_pb::secret::PbSecretRef;
+
+use crate::sink::catalog::SinkFormatDesc;
 use crate::source::cdc::external::CdcTableType;
 use crate::source::iceberg::ICEBERG_CONNECTOR;
 use crate::source::{
@@ -110,6 +113,16 @@ pub trait WithPropertiesExt: Get + Sized {
         self.is_cdc_connector() && CdcTableType::from_properties(self).can_backfill()
     }
 
+    /// Tables with MySQL and PostgreSQL connectors are maintained for backward compatibility.
+    /// The newly added SQL Server CDC connector is only supported when created as shared.
+    fn is_shareable_only_cdc_connector(&self) -> bool {
+        self.is_cdc_connector() && CdcTableType::from_properties(self).shareable_only()
+    }
+
+    fn enable_transaction_metadata(&self) -> bool {
+        CdcTableType::from_properties(self).enable_transaction_metadata()
+    }
+
     #[inline(always)]
     fn is_iceberg_connector(&self) -> bool {
         let Some(connector) = self.get_connector() else {
@@ -135,3 +148,67 @@ pub trait WithPropertiesExt: Get + Sized {
 }
 
 impl<T: Get> WithPropertiesExt for T {}
+
+/// Options or properties extracted from the `WITH` clause of DDLs.
+#[derive(Default, Clone, Debug, PartialEq, Eq, Hash)]
+pub struct WithOptionsSecResolved {
+    inner: BTreeMap<String, String>,
+    secret_ref: BTreeMap<String, PbSecretRef>,
+}
+
+impl std::ops::Deref for WithOptionsSecResolved {
+    type Target = BTreeMap<String, String>;
+
+    fn deref(&self) -> &Self::Target {
+        &self.inner
+    }
+}
+
+impl std::ops::DerefMut for WithOptionsSecResolved {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.inner
+    }
+}
+
+impl WithOptionsSecResolved {
+    /// Create a new [`WithOptions`] from a option [`BTreeMap`] and resolved secret ref.
+    pub fn new(inner: BTreeMap<String, String>, secret_ref: BTreeMap<String, PbSecretRef>) -> Self {
+        Self { inner, secret_ref }
+    }
+
+    /// Create a new [`WithOptions`] from a [`BTreeMap`].
+    pub fn without_secrets(inner: BTreeMap<String, String>) -> Self {
+        Self {
+            inner,
+            secret_ref: Default::default(),
+        }
+    }
+
+    /// Take the value of the option map and secret refs.
+    pub fn into_parts(self) -> (BTreeMap<String, String>, BTreeMap<String, PbSecretRef>) {
+        (self.inner, self.secret_ref)
+    }
+
+    pub fn value_eq_ignore_case(&self, key: &str, val: &str) -> bool {
+        if let Some(inner_val) = self.inner.get(key) {
+            if inner_val.eq_ignore_ascii_case(val) {
+                return true;
+            }
+        }
+        false
+    }
+}
+
+/// For `planner_test` crate so that it does not depend directly on `connector` crate just for `SinkFormatDesc`.
+impl TryFrom<&WithOptionsSecResolved> for Option<SinkFormatDesc> {
+    type Error = crate::sink::SinkError;
+
+    fn try_from(value: &WithOptionsSecResolved) -> std::result::Result<Self, Self::Error> {
+        let connector = value.get(crate::sink::CONNECTOR_TYPE_KEY);
+        let r#type = value.get(crate::sink::SINK_TYPE_OPTION);
+        match (connector, r#type) {
+            (Some(c), Some(t)) => SinkFormatDesc::from_legacy_type(c, t),
+            _ => Ok(None),
+        }
+    }
+}
diff --git a/src/connector/with_options_sink.yaml b/src/connector/with_options_sink.yaml
index 27362900dd054..653acaadaaaf1 100644
--- a/src/connector/with_options_sink.yaml
+++ b/src/connector/with_options_sink.yaml
@@ -749,54 +749,36 @@ RedisConfig:
     required: true
 SnowflakeConfig:
   fields:
-  - name: snowflake.database
-    field_type: String
-    comments: The snowflake database used for sinking
-    required: true
-  - name: snowflake.schema
-    field_type: String
-    comments: The corresponding schema where sink table exists
-    required: true
-  - name: snowflake.pipe
-    field_type: String
-    comments: The created pipe object, will be used as `insertFiles` target
-    required: true
-  - name: snowflake.account_identifier
-    field_type: String
-    comments: 'The unique, snowflake provided `account_identifier`  NOTE: please use the form `<orgname>-<account_name>`  For detailed guidance, reference: <https://docs.snowflake.com/en/user-guide/admin-account-identifier>'
-    required: true
-  - name: snowflake.user
-    field_type: String
-    comments: 'The user that owns the table to be sinked  NOTE: the user should''ve been granted corresponding *role*  reference: <https://docs.snowflake.com/en/sql-reference/sql/grant-role>'
-    required: true
-  - name: snowflake.rsa_public_key_fp
-    field_type: String
-    comments: 'The public key fingerprint used when generating custom `jwt_token`  reference: <https://docs.snowflake.com/en/developer-guide/sql-api/authenticating>'
-    required: true
-  - name: snowflake.private_key
-    field_type: String
-    comments: The rsa pem key *without* encryption
-    required: true
   - name: snowflake.s3_bucket
     field_type: String
     comments: The s3 bucket where intermediate sink files will be stored
     required: true
+    alias:
+    - s3.bucket_name
   - name: snowflake.s3_path
     field_type: String
     comments: The optional s3 path to be specified  the actual file location would be `s3://<s3_bucket>/<s3_path>/<rw_auto_gen_intermediate_file_name>`  if this field is specified by user(s)  otherwise it would be `s3://<s3_bucket>/<rw_auto_gen_intermediate_file_name>`
     required: false
+    alias:
+    - s3.path
   - name: snowflake.aws_access_key_id
     field_type: String
     comments: s3 credentials
     required: true
+    alias:
+    - s3.credentials.access
   - name: snowflake.aws_secret_access_key
     field_type: String
     comments: s3 credentials
     required: true
+    alias:
+    - s3.credentials.secret
   - name: snowflake.aws_region
     field_type: String
     comments: The s3 region, e.g., us-east-2
     required: true
+    alias:
+    - s3.region_name
 SqlServerConfig:
   fields:
   - name: sqlserver.host
diff --git a/src/ctl/src/cmd_impl/await_tree.rs b/src/ctl/src/cmd_impl/await_tree.rs
index 1c4ff98562791..00d8d8461d340 100644
--- a/src/ctl/src/cmd_impl/await_tree.rs
+++ b/src/ctl/src/cmd_impl/await_tree.rs
@@ -28,7 +28,7 @@ pub async fn dump(context: &CtlContext) -> anyhow::Result<()> {
     let compute_nodes = meta_client
         .list_worker_nodes(Some(WorkerType::ComputeNode))
         .await?;
-    let clients = ComputeClientPool::default();
+    let clients = ComputeClientPool::adhoc();
 
     // FIXME: the compute node may not be accessible directly from risectl, we may let the meta
     // service collect the reports from all compute nodes in the future.
diff --git a/src/ctl/src/cmd_impl/hummock/list_version.rs b/src/ctl/src/cmd_impl/hummock/list_version.rs
index 1c7ea9c40936a..0f88d881ddb69 100644
--- a/src/ctl/src/cmd_impl/hummock/list_version.rs
+++ b/src/ctl/src/cmd_impl/hummock/list_version.rs
@@ -30,11 +30,11 @@ pub async fn list_version(
     } else if verbose {
         version.levels.iter_mut().for_each(|(_cg_id, levels)| {
             // l0
-            if levels.l0.is_some() {
-                let l0 = levels.l0.as_mut().unwrap();
+            {
+                let l0 = &mut levels.l0;
                 for sub_level in &mut l0.sub_levels {
                     for t in &mut sub_level.table_infos {
-                        t.key_range = None;
+                        t.remove_key_range();
                     }
                 }
             }
@@ -42,7 +42,7 @@ pub async fn list_version(
             // l1 ~ lmax
             for level in &mut levels.levels {
                 for t in &mut level.table_infos {
-                    t.key_range = None;
+                    t.remove_key_range();
                 }
             }
         });
@@ -58,23 +58,23 @@ pub async fn list_version(
             println!("CompactionGroup {}", cg);
 
             // l0
-            if levels.l0.is_some() {
-                for sub_level in levels.l0.as_ref().unwrap().sub_levels.iter().rev() {
+            {
+                for sub_level in levels.l0.sub_levels.iter().rev() {
                     println!(
                         "sub_level_id {} type {} sst_num {} size {}",
                         sub_level.sub_level_id,
-                        sub_level.level_type().as_str_name(),
+                        sub_level.level_type.as_str_name(),
                         sub_level.table_infos.len(),
                         sub_level.total_file_size
                     )
                 }
             }
 
-            for level in levels.get_levels() {
+            for level in &levels.levels {
                 println!(
                     "level_idx {} type {} sst_num {} size {}",
                     level.level_idx,
-                    level.level_type().as_str_name(),
+                    level.level_type.as_str_name(),
                     level.table_infos.len(),
                     level.total_file_size
                 )
diff --git a/src/ctl/src/cmd_impl/hummock/sst_dump.rs b/src/ctl/src/cmd_impl/hummock/sst_dump.rs
index 3a71fbd007214..03a7b35a85192 100644
--- a/src/ctl/src/cmd_impl/hummock/sst_dump.rs
+++ b/src/ctl/src/cmd_impl/hummock/sst_dump.rs
@@ -28,9 +28,10 @@ use risingwave_common::util::value_encoding::column_aware_row_encoding::ColumnAw
 use risingwave_common::util::value_encoding::{BasicSerde, EitherSerde, ValueRowDeserializer};
 use risingwave_frontend::TableCatalog;
 use risingwave_hummock_sdk::key::FullKey;
+use risingwave_hummock_sdk::level::Level;
+use risingwave_hummock_sdk::sstable_info::SstableInfo;
 use risingwave_hummock_sdk::HummockSstableObjectId;
 use risingwave_object_store::object::{ObjectMetadata, ObjectStoreImpl};
-use risingwave_pb::hummock::{Level, SstableInfo};
 use risingwave_rpc_client::MetaClient;
 use risingwave_storage::hummock::value::HummockValue;
 use risingwave_storage::hummock::{
@@ -83,11 +84,11 @@ pub async fn sst_dump(context: &CtlContext, args: SstDumpArgs) -> anyhow::Result
         for level in version.get_combined_levels() {
             for sstable_info in &level.table_infos {
                 if let Some(object_id) = &args.object_id {
-                    if *object_id == sstable_info.get_object_id() {
+                    if *object_id == sstable_info.object_id {
                         print_level(level, sstable_info);
                         sst_dump_via_sstable_store(
                             &sstable_store,
-                            sstable_info.get_object_id(),
+                            sstable_info.object_id,
                             sstable_info.meta_offset,
                             sstable_info.file_size,
                             &table_data,
@@ -100,7 +101,7 @@ pub async fn sst_dump(context: &CtlContext, args: SstDumpArgs) -> anyhow::Result
                     print_level(level, sstable_info);
                     sst_dump_via_sstable_store(
                         &sstable_store,
-                        sstable_info.get_object_id(),
+                        sstable_info.object_id,
                         sstable_info.meta_offset,
                         sstable_info.file_size,
                         &table_data,
@@ -137,7 +138,7 @@ pub async fn sst_dump(context: &CtlContext, args: SstDumpArgs) -> anyhow::Result
             .await?;
         } else {
             let mut metadata_iter = sstable_store
-                .list_object_metadata_from_object_store()
+                .list_object_metadata_from_object_store(None)
                 .await?;
             while let Some(obj) = metadata_iter.try_next().await? {
                 print_object(&obj);
@@ -161,7 +162,7 @@ pub async fn sst_dump(context: &CtlContext, args: SstDumpArgs) -> anyhow::Result
 }
 
 fn print_level(level: &Level, sst_info: &SstableInfo) {
-    println!("Level Type: {}", level.level_type);
+    println!("Level Type: {}", level.level_type.as_str_name());
     println!("Level Idx: {}", level.level_idx);
     if level.level_idx == 0 {
         println!("L0 Sub-Level Idx: {}", level.sub_level_id);
diff --git a/src/ctl/src/cmd_impl/hummock/trigger_full_gc.rs b/src/ctl/src/cmd_impl/hummock/trigger_full_gc.rs
index f5e90eba48d88..bded39fa2adb8 100644
--- a/src/ctl/src/cmd_impl/hummock/trigger_full_gc.rs
+++ b/src/ctl/src/cmd_impl/hummock/trigger_full_gc.rs
@@ -19,9 +19,12 @@ use crate::CtlContext;
 pub async fn trigger_full_gc(
     context: &CtlContext,
     sst_retention_time_sec: u64,
+    prefix: Option<String>,
 ) -> anyhow::Result<()> {
     let meta_client = context.meta_client().await?;
-    let result = meta_client.trigger_full_gc(sst_retention_time_sec).await;
+    let result = meta_client
+        .trigger_full_gc(sst_retention_time_sec, prefix)
+        .await;
     println!("{:#?}", result);
     Ok(())
 }
diff --git a/src/ctl/src/cmd_impl/hummock/validate_version.rs b/src/ctl/src/cmd_impl/hummock/validate_version.rs
index e8f61a8c98358..4d857ad8b91f1 100644
--- a/src/ctl/src/cmd_impl/hummock/validate_version.rs
+++ b/src/ctl/src/cmd_impl/hummock/validate_version.rs
@@ -20,11 +20,12 @@ use itertools::Itertools;
 use risingwave_common::util::epoch::Epoch;
 use risingwave_hummock_sdk::compaction_group::hummock_version_ext;
 use risingwave_hummock_sdk::key::{FullKey, UserKey};
+use risingwave_hummock_sdk::sstable_info::SstableInfo;
 use risingwave_hummock_sdk::version::{HummockVersion, HummockVersionDelta};
 use risingwave_hummock_sdk::{version_archive_dir, HummockSstableObjectId, HummockVersionId};
 use risingwave_object_store::object::ObjectStoreRef;
 use risingwave_pb::hummock::group_delta::DeltaType;
-use risingwave_pb::hummock::{HummockVersionArchive, SstableInfo};
+use risingwave_pb::hummock::HummockVersionArchive;
 use risingwave_rpc_client::HummockMetaClient;
 use risingwave_storage::hummock::value::HummockValue;
 use risingwave_storage::hummock::{Block, BlockHolder, BlockIterator, SstableStoreRef};
@@ -99,18 +100,9 @@ async fn print_user_key_in_version(
 ) -> anyhow::Result<()> {
     println!("print key {:?} in version {}", target_key, version.id);
     for cg in version.levels.values() {
-        for level in cg
-            .l0
-            .as_ref()
-            .unwrap()
-            .sub_levels
-            .iter()
-            .rev()
-            .chain(cg.levels.iter())
-        {
+        for level in cg.l0.sub_levels.iter().rev().chain(cg.levels.iter()) {
             for sstable_info in &level.table_infos {
-                use risingwave_hummock_sdk::key_range::KeyRange;
-                let key_range: KeyRange = sstable_info.key_range.as_ref().unwrap().into();
+                let key_range = &sstable_info.key_range;
                 let left_user_key = FullKey::decode(&key_range.left);
                 let right_user_key = FullKey::decode(&key_range.right);
                 if left_user_key.user_key > *target_key || *target_key > right_user_key.user_key {
diff --git a/src/ctl/src/cmd_impl/meta/cluster_info.rs b/src/ctl/src/cmd_impl/meta/cluster_info.rs
index 7d38fe8f3ae4e..cbc21ca6ec610 100644
--- a/src/ctl/src/cmd_impl/meta/cluster_info.rs
+++ b/src/ctl/src/cmd_impl/meta/cluster_info.rs
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::{BTreeMap, HashMap};
+use std::collections::{BTreeMap, BTreeSet, HashMap};
 
 use comfy_table::{Attribute, Cell, Row, Table};
 use itertools::Itertools;
@@ -88,7 +88,7 @@ pub async fn cluster_info(context: &CtlContext) -> anyhow::Result<()> {
         revision,
     } = get_cluster_info(context).await?;
 
-    // Fragment ID -> [Parallel Unit ID -> (Parallel Unit, Actor)]
+    // Fragment ID -> [Worker ID -> [Actor ID]]
     let mut fragments = BTreeMap::new();
     // Fragment ID -> Table Fragments' State
     let mut fragment_states = HashMap::new();
@@ -96,32 +96,23 @@ pub async fn cluster_info(context: &CtlContext) -> anyhow::Result<()> {
     for table_fragment in &table_fragments {
         for (&id, fragment) in &table_fragment.fragments {
             for actor in &fragment.actors {
-                let parallel_unit = table_fragment
+                let worker_id = table_fragment
                     .actor_status
                     .get(&actor.actor_id)
                     .unwrap()
-                    .get_parallel_unit()
-                    .unwrap();
+                    .worker_id();
+
                 fragments
                     .entry(id)
-                    .or_insert_with(HashMap::new)
-                    .insert(parallel_unit.id, (parallel_unit, actor));
+                    .or_insert_with(BTreeMap::new)
+                    .entry(worker_id)
+                    .or_insert(BTreeSet::new())
+                    .insert(actor.actor_id);
             }
             fragment_states.insert(id, table_fragment.state());
         }
     }
 
-    // Parallel Unit ID -> Worker Node
-    let all_parallel_units: BTreeMap<_, _> = worker_nodes
-        .iter()
-        .flat_map(|worker_node| {
-            worker_node
-                .parallel_units
-                .iter()
-                .map(|parallel_unit| (parallel_unit.id, worker_node.clone()))
-        })
-        .collect();
-
     let mut table = Table::new();
 
     let cross_out_if_creating = |cell: Cell, fid: u32| -> Cell {
@@ -132,11 +123,10 @@ pub async fn cluster_info(context: &CtlContext) -> anyhow::Result<()> {
         }
     };
 
-    // Compute Node, Parallel Unit, Frag 1, Frag 2, ..., Frag N
+    // Compute Node, Frag 1, Frag 2, ..., Frag N
     table.set_header({
         let mut row = Row::new();
         row.add_cell("Compute Node".into());
-        row.add_cell("Parallel Unit".into());
         for &fid in fragments.keys() {
             let cell = Cell::new(format!("Frag {fid}"));
             let cell = cross_out_if_creating(cell, fid);
@@ -146,8 +136,8 @@ pub async fn cluster_info(context: &CtlContext) -> anyhow::Result<()> {
     });
 
     let mut last_worker_id = None;
-    for (pu, worker) in all_parallel_units {
-        // Compute Node, Parallel Unit, Actor 1, Actor 11, -, ..., Actor N
+    for worker in worker_nodes {
+        // Compute Node,  Actor 1, Actor 11, -, ..., Actor N
         let mut row = Row::new();
         row.add_cell(if last_worker_id == Some(worker.id) {
             "".into()
@@ -166,14 +156,17 @@ pub async fn cluster_info(context: &CtlContext) -> anyhow::Result<()> {
             ))
             .add_attribute(Attribute::Bold)
         });
-        row.add_cell(pu.into());
-        for (&fid, f) in &fragments {
-            let cell = if let Some((_pu, actor)) = f.get(&pu) {
-                actor.actor_id.into()
+        for (&fragment_id, worker_actors) in &fragments {
+            let cell = if let Some(actors) = worker_actors.get(&worker.id) {
+                actors
+                    .iter()
+                    .map(|actor| format!("{}", actor))
+                    .join(",")
+                    .into()
             } else {
                 "-".into()
             };
-            let cell = cross_out_if_creating(cell, fid);
+            let cell = cross_out_if_creating(cell, fragment_id);
             row.add_cell(cell);
         }
         table.add_row(row);
diff --git a/src/ctl/src/cmd_impl/meta/migration.rs b/src/ctl/src/cmd_impl/meta/migration.rs
index 614fcd6be225c..ed82f57af2af1 100644
--- a/src/ctl/src/cmd_impl/meta/migration.rs
+++ b/src/ctl/src/cmd_impl/meta/migration.rs
@@ -156,18 +156,12 @@ pub async fn migrate(from: EtcdBackend, target: String, force_clean: bool) -> an
             .await?;
         if worker.worker_type() == WorkerType::ComputeNode {
             let pb_property = worker.worker_node.property.as_ref().unwrap();
-            let parallel_unit_ids = worker
-                .worker_node
-                .parallel_units
-                .iter()
-                .map(|pu| pu.id as i32)
-                .collect_vec();
             let property = worker_property::ActiveModel {
                 worker_id: Set(worker.worker_id() as _),
-                parallel_unit_ids: Set(parallel_unit_ids.into()),
                 is_streaming: Set(pb_property.is_streaming),
                 is_serving: Set(pb_property.is_serving),
                 is_unschedulable: Set(pb_property.is_unschedulable),
+                parallelism: Set(worker.worker_node.parallelism() as _),
             };
             WorkerProperty::insert(property)
                 .exec(&meta_store_sql.conn)
@@ -748,7 +742,7 @@ pub async fn migrate(from: EtcdBackend, target: String, force_clean: bool) -> an
                     id: Set(vd.id as _),
                     prev_id: Set(vd.prev_id as _),
                     max_committed_epoch: Set(vd.max_committed_epoch as _),
-                    safe_epoch: Set(vd.safe_epoch as _),
+                    safe_epoch: Set(vd.visible_table_safe_epoch() as _),
                     trivial_move: Set(vd.trivial_move),
                     full_version_delta: Set((&vd.to_protobuf()).into()),
                 })
diff --git a/src/ctl/src/cmd_impl/meta/reschedule.rs b/src/ctl/src/cmd_impl/meta/reschedule.rs
index 8d0a45be842fb..505343fc95db8 100644
--- a/src/ctl/src/cmd_impl/meta/reschedule.rs
+++ b/src/ctl/src/cmd_impl/meta/reschedule.rs
@@ -15,14 +15,13 @@
 use std::collections::{HashMap, HashSet};
 use std::process::exit;
 
-use anyhow::{anyhow, Error, Result};
+use anyhow::{anyhow, Result};
 use inquire::Confirm;
 use itertools::Itertools;
-use regex::{Match, Regex};
+use regex::Regex;
+use risingwave_meta::manager::WorkerId;
 use risingwave_pb::common::WorkerNode;
-use risingwave_pb::meta::get_reschedule_plan_request::PbPolicy;
-use risingwave_pb::meta::table_fragments::ActorStatus;
-use risingwave_pb::meta::{GetClusterInfoResponse, GetReschedulePlanResponse, Reschedule};
+use risingwave_pb::meta::{GetClusterInfoResponse, PbWorkerReschedule};
 use serde::{Deserialize, Serialize};
 use thiserror_ext::AsReport;
 
@@ -34,16 +33,13 @@ pub struct ReschedulePayload {
     pub reschedule_revision: u64,
 
     #[serde(rename = "reschedule_plan")]
-    pub reschedule_plan: HashMap<u32, FragmentReschedulePlan>,
+    pub worker_reschedule_plan: HashMap<u32, WorkerReschedulePlan>,
 }
 
 #[derive(Serialize, Deserialize, Debug)]
-pub struct FragmentReschedulePlan {
-    #[serde(rename = "added_parallel_units")]
-    pub added_parallel_units: Vec<u32>,
-
-    #[serde(rename = "removed_parallel_units")]
-    pub removed_parallel_units: Vec<u32>,
+pub struct WorkerReschedulePlan {
+    #[serde(rename = "actor_count_diff")]
+    pub actor_count_diff: HashMap<WorkerId, i32>,
 }
 
 #[derive(Debug)]
@@ -52,66 +48,34 @@ pub enum RescheduleInput {
     FilePath(String),
 }
 
-impl From<FragmentReschedulePlan> for Reschedule {
-    fn from(value: FragmentReschedulePlan) -> Self {
-        let FragmentReschedulePlan {
-            added_parallel_units,
-            removed_parallel_units,
-        } = value;
+impl From<WorkerReschedulePlan> for PbWorkerReschedule {
+    fn from(value: WorkerReschedulePlan) -> Self {
+        let WorkerReschedulePlan { actor_count_diff } = value;
 
-        Reschedule {
-            added_parallel_units,
-            removed_parallel_units,
+        PbWorkerReschedule {
+            worker_actor_diff: actor_count_diff
+                .into_iter()
+                .map(|(k, v)| (k as _, v as _))
+                .collect(),
         }
     }
 }
 
-impl From<Reschedule> for FragmentReschedulePlan {
-    fn from(value: Reschedule) -> Self {
-        let Reschedule {
-            added_parallel_units,
-            removed_parallel_units,
+impl From<PbWorkerReschedule> for WorkerReschedulePlan {
+    fn from(value: PbWorkerReschedule) -> Self {
+        let PbWorkerReschedule {
+            worker_actor_diff: actor_count_diff,
         } = value;
 
-        FragmentReschedulePlan {
-            added_parallel_units,
-            removed_parallel_units,
+        WorkerReschedulePlan {
+            actor_count_diff: actor_count_diff
+                .into_iter()
+                .map(|(k, v)| (k as _, v as _))
+                .collect(),
         }
     }
 }
 
-const RESCHEDULE_MATCH_REGEXP: &str =
-    r"^(?P<fragment>\d+)(?:-\[(?P<removed>\d+(?:,\d+)*)])?(?:\+\[(?P<added>\d+(?:,\d+)*)])?$";
-const RESCHEDULE_FRAGMENT_KEY: &str = "fragment";
-const RESCHEDULE_REMOVED_KEY: &str = "removed";
-const RESCHEDULE_ADDED_KEY: &str = "added";
-
-// For plan `100-[1,2,3]+[4,5];101-[1];102+[3]`, the following reschedule request will be generated
-// {
-//     100: Reschedule {
-//         added_parallel_units: [
-//             4,
-//             5,
-//         ],
-//         removed_parallel_units: [
-//             1,
-//             2,
-//             3,
-//         ],
-//     },
-//     101: Reschedule {
-//         added_parallel_units: [],
-//         removed_parallel_units: [
-//             1,
-//         ],
-//     },
-//     102: Reschedule {
-//         added_parallel_units: [
-//             3,
-//         ],
-//         removed_parallel_units: [],
-//     },
-// }
 pub async fn reschedule(
     context: &CtlContext,
     plan: Option<String>,
@@ -128,13 +92,13 @@ pub async fn reschedule(
             let file = std::fs::File::open(path)?;
             let ReschedulePayload {
                 reschedule_revision,
-                reschedule_plan,
+                worker_reschedule_plan,
             } = serde_yaml::from_reader(file)?;
             (
-                reschedule_plan
+                worker_reschedule_plan
                     .into_iter()
-                    .map(|(fragment_id, fragment_reschedule_plan)| {
-                        (fragment_id, fragment_reschedule_plan.into())
+                    .map(|(fragment_id, worker_reschedule_plan)| {
+                        (fragment_id, worker_reschedule_plan.into())
                     })
                     .collect(),
                 reschedule_revision,
@@ -143,14 +107,14 @@ pub async fn reschedule(
         _ => unreachable!(),
     };
 
+    if reschedules.is_empty() {
+        return Ok(());
+    }
+
     for (fragment_id, reschedule) in &reschedules {
         println!("For fragment #{}", fragment_id);
-        if !reschedule.removed_parallel_units.is_empty() {
-            println!("\tRemove: {:?}", reschedule.removed_parallel_units);
-        }
-
-        if !reschedule.added_parallel_units.is_empty() {
-            println!("\tAdd:    {:?}", reschedule.added_parallel_units);
+        if !reschedule.get_worker_actor_diff().is_empty() {
+            println!("\tChange: {:?}", reschedule.get_worker_actor_diff());
         }
 
         println!();
@@ -177,64 +141,50 @@ pub async fn reschedule(
     Ok(())
 }
 
-fn parse_plan(mut plan: String) -> Result<HashMap<u32, Reschedule>, Error> {
+// It will match formats like `1:[1:+1,2:-1,3:1];2:[1:1,2:1]`, indicating which workers' actors need to change in quantity for each fragment.
+fn parse_plan(mut plan: String) -> Result<HashMap<u32, PbWorkerReschedule>> {
     let mut reschedules = HashMap::new();
-
-    let regex = Regex::new(RESCHEDULE_MATCH_REGEXP)?;
-
+    let regex = Regex::new(r"^(\d+):\[((?:\d+:[+-]?\d+,?)+)]$")?;
     plan.retain(|c| !c.is_whitespace());
 
     for fragment_reschedule_plan in plan.split(';') {
+        if fragment_reschedule_plan.is_empty() {
+            continue;
+        }
+
         let captures = regex
             .captures(fragment_reschedule_plan)
             .ok_or_else(|| anyhow!("plan \"{}\" format illegal", fragment_reschedule_plan))?;
 
         let fragment_id = captures
-            .name(RESCHEDULE_FRAGMENT_KEY)
+            .get(1)
             .and_then(|mat| mat.as_str().parse::<u32>().ok())
             .ok_or_else(|| anyhow!("plan \"{}\" does not have a valid fragment id", plan))?;
 
-        let split_fn = |mat: Match<'_>| {
-            mat.as_str()
-                .split(',')
-                .map(|id_str| id_str.parse::<u32>().map_err(Error::msg))
-                .collect::<Result<Vec<_>>>()
-        };
+        let worker_changes: Vec<&str> = captures[2].split(',').collect();
 
-        let removed_parallel_units = captures
-            .name(RESCHEDULE_REMOVED_KEY)
-            .map(split_fn)
-            .transpose()?
-            .unwrap_or_default();
-        let added_parallel_units = captures
-            .name(RESCHEDULE_ADDED_KEY)
-            .map(split_fn)
-            .transpose()?
-            .unwrap_or_default();
-
-        if !(removed_parallel_units.is_empty() && added_parallel_units.is_empty()) {
-            reschedules.insert(
-                fragment_id,
-                Reschedule {
-                    added_parallel_units,
-                    removed_parallel_units,
-                },
-            );
+        let mut worker_actor_diff = HashMap::new();
+        for worker_change in &worker_changes {
+            let (worker_id, count) = worker_change
+                .split(':')
+                .map(|v| v.parse::<i32>().unwrap())
+                .collect_tuple::<(_, _)>()
+                .unwrap();
+
+            if let Some(dup_change) = worker_actor_diff.insert(worker_id as u32, count) {
+                anyhow::bail!(
+                    "duplicate worker id {worker_id} in plan, prev {worker_id} -> {dup_change}",
+                );
+            }
+        }
+
+        if !worker_actor_diff.is_empty() {
+            reschedules.insert(fragment_id, PbWorkerReschedule { worker_actor_diff });
         }
     }
     Ok(reschedules)
 }
 
-pub async fn get_reschedule_plan(
-    context: &CtlContext,
-    policy: PbPolicy,
-    revision: u64,
-) -> Result<GetReschedulePlanResponse> {
-    let meta_client = context.meta_client().await?;
-    let response = meta_client.get_reschedule_plan(policy, revision).await?;
-    Ok(response)
-}
-
 pub async fn unregister_workers(
     context: &CtlContext,
     workers: Vec<String>,
@@ -314,9 +264,8 @@ pub async fn unregister_workers(
                     table_fragments
                         .actor_status
                         .get(&actor.actor_id)
-                        .and_then(|ActorStatus { parallel_unit, .. }| parallel_unit.clone())
+                        .map(|actor_status| actor_status.worker_id())
                         .unwrap()
-                        .worker_node_id
                 })
                 .collect();
 
diff --git a/src/ctl/src/cmd_impl/profile.rs b/src/ctl/src/cmd_impl/profile.rs
index edb48df5aeb39..bc7897a74c11c 100644
--- a/src/ctl/src/cmd_impl/profile.rs
+++ b/src/ctl/src/cmd_impl/profile.rs
@@ -33,7 +33,7 @@ pub async fn cpu_profile(context: &CtlContext, sleep_s: u64) -> anyhow::Result<(
         .into_iter()
         .filter(|w| w.r#type() == WorkerType::ComputeNode);
 
-    let clients = ComputeClientPool::default();
+    let clients = ComputeClientPool::adhoc();
 
     let profile_root_path = std::env::var("PREFIX_PROFILING").unwrap_or_else(|_| {
         tracing::info!("PREFIX_PROFILING is not set, using current directory");
@@ -96,7 +96,7 @@ pub async fn heap_profile(context: &CtlContext, dir: Option<String>) -> anyhow::
         .into_iter()
         .filter(|w| w.r#type() == WorkerType::ComputeNode);
 
-    let clients = ComputeClientPool::default();
+    let clients = ComputeClientPool::adhoc();
 
     let mut profile_futs = vec![];
 
diff --git a/src/ctl/src/cmd_impl/scale/resize.rs b/src/ctl/src/cmd_impl/scale/resize.rs
index 6b990c7865519..364426e7996eb 100644
--- a/src/ctl/src/cmd_impl/scale/resize.rs
+++ b/src/ctl/src/cmd_impl/scale/resize.rs
@@ -13,22 +13,14 @@
 // limitations under the License.
 
 use std::collections::{HashMap, HashSet};
-use std::ops::Sub;
 use std::process::exit;
 
-use inquire::Confirm;
 use itertools::Itertools;
-use risingwave_pb::meta::get_reschedule_plan_request::{
-    PbPolicy, StableResizePolicy, WorkerChanges,
-};
 use risingwave_pb::meta::update_worker_node_schedulability_request::Schedulability;
-use risingwave_pb::meta::{GetClusterInfoResponse, GetReschedulePlanResponse};
-use risingwave_stream::task::FragmentId;
+use risingwave_pb::meta::GetClusterInfoResponse;
 use thiserror_ext::AsReport;
 
-use crate::cmd_impl::meta::ReschedulePayload;
 use crate::common::CtlContext;
-use crate::{ScaleCommon, ScaleHorizonCommands, ScaleVerticalCommands};
 
 macro_rules! fail {
     ($($arg:tt)*) => {{
@@ -37,350 +29,6 @@ macro_rules! fail {
     }};
 }
 
-impl From<ScaleHorizonCommands> for ScaleCommandContext {
-    fn from(value: ScaleHorizonCommands) -> Self {
-        let ScaleHorizonCommands {
-            exclude_workers,
-            include_workers,
-            target_parallelism,
-            common:
-                ScaleCommon {
-                    generate,
-                    output,
-                    yes,
-                    fragments,
-                },
-        } = value;
-
-        Self {
-            exclude_workers,
-            include_workers,
-            target_parallelism,
-            generate,
-            output,
-            yes,
-            fragments,
-            target_parallelism_per_worker: None,
-            exclusive_for_vertical: false,
-        }
-    }
-}
-
-impl From<ScaleVerticalCommands> for ScaleCommandContext {
-    fn from(value: ScaleVerticalCommands) -> Self {
-        let ScaleVerticalCommands {
-            workers,
-            target_parallelism_per_worker,
-            common:
-                ScaleCommon {
-                    generate,
-                    output,
-                    yes,
-                    fragments,
-                },
-            exclusive,
-        } = value;
-
-        Self {
-            exclude_workers: None,
-            include_workers: workers,
-            target_parallelism: None,
-            generate,
-            output,
-            yes,
-            fragments,
-            target_parallelism_per_worker,
-            exclusive_for_vertical: exclusive,
-        }
-    }
-}
-
-pub struct ScaleCommandContext {
-    exclude_workers: Option<Vec<String>>,
-    include_workers: Option<Vec<String>>,
-    target_parallelism: Option<u32>,
-    generate: bool,
-    output: Option<String>,
-    yes: bool,
-    fragments: Option<Vec<u32>>,
-    target_parallelism_per_worker: Option<u32>,
-    exclusive_for_vertical: bool,
-}
-
-pub async fn resize(ctl_ctx: &CtlContext, scale_ctx: ScaleCommandContext) -> anyhow::Result<()> {
-    let meta_client = ctl_ctx.meta_client().await?;
-
-    let GetClusterInfoResponse {
-        worker_nodes,
-        table_fragments,
-        actor_splits: _actor_splits,
-        source_infos: _source_infos,
-        revision,
-    } = match meta_client.get_cluster_info().await {
-        Ok(resp) => resp,
-        Err(e) => {
-            fail!("Failed to fetch cluster info: {}", e.as_report());
-        }
-    };
-
-    if worker_nodes.is_empty() {
-        println!("No worker nodes found");
-        return Ok(());
-    }
-
-    if table_fragments.is_empty() {
-        println!("No tables found");
-        return Ok(());
-    }
-
-    println!("Cluster info fetched, revision: {}", revision);
-    println!("Worker nodes: {}", worker_nodes.len());
-
-    let streaming_workers_index_by_id = worker_nodes
-        .into_iter()
-        .filter(|worker| {
-            worker
-                .property
-                .as_ref()
-                .map(|property| property.is_streaming)
-                .unwrap_or(false)
-        })
-        .map(|worker| (worker.id, worker))
-        .collect::<HashMap<_, _>>();
-
-    let streaming_workers_index_by_host = streaming_workers_index_by_id
-        .values()
-        .map(|worker| {
-            let host = worker.get_host().expect("worker host must be set");
-            (format!("{}:{}", host.host, host.port), worker.clone())
-        })
-        .collect::<HashMap<_, _>>();
-
-    let worker_input_to_worker_ids = |inputs: Vec<String>, support_all: bool| -> Vec<u32> {
-        let mut result: HashSet<_> = HashSet::new();
-
-        if inputs.len() == 1 && inputs[0].to_lowercase() == "all" && support_all {
-            return streaming_workers_index_by_id.keys().cloned().collect();
-        }
-
-        for input in inputs {
-            let worker_id = input.parse::<u32>().ok().or_else(|| {
-                streaming_workers_index_by_host
-                    .get(&input)
-                    .map(|worker| worker.id)
-            });
-
-            if let Some(worker_id) = worker_id {
-                if !result.insert(worker_id) {
-                    println!("warn: {} and {} are the same worker", input, worker_id);
-                }
-            } else {
-                fail!("Invalid worker input: {}", input);
-            }
-        }
-
-        result.into_iter().collect()
-    };
-
-    println!(
-        "Streaming workers found: {}",
-        streaming_workers_index_by_id.len()
-    );
-
-    let ScaleCommandContext {
-        exclude_workers,
-        include_workers,
-        target_parallelism,
-        target_parallelism_per_worker,
-        generate,
-        output,
-        yes,
-        fragments,
-        exclusive_for_vertical,
-    } = scale_ctx;
-
-    let worker_changes = {
-        let mut exclude_worker_ids =
-            worker_input_to_worker_ids(exclude_workers.unwrap_or_default(), false);
-        let include_worker_ids =
-            worker_input_to_worker_ids(include_workers.unwrap_or_default(), true);
-
-        match (target_parallelism, target_parallelism_per_worker) {
-            (Some(_), Some(_)) => {
-                fail!("Cannot specify both target parallelism and target parallelism per worker")
-            }
-            (_, Some(_)) if include_worker_ids.is_empty() => {
-                fail!("Cannot specify target parallelism per worker without including any worker")
-            }
-            (Some(0), _) => fail!("Target parallelism must be greater than 0"),
-            _ => {}
-        }
-
-        for worker_id in exclude_worker_ids.iter().chain(include_worker_ids.iter()) {
-            if !streaming_workers_index_by_id.contains_key(worker_id) {
-                fail!("Invalid worker id: {}", worker_id);
-            }
-        }
-
-        for include_worker_id in &include_worker_ids {
-            let worker_is_unschedulable = streaming_workers_index_by_id
-                .get(include_worker_id)
-                .and_then(|worker| worker.property.as_ref())
-                .map(|property| property.is_unschedulable)
-                .unwrap_or(false);
-
-            if worker_is_unschedulable {
-                fail!(
-                    "Worker {} is unschedulable, should not be included",
-                    include_worker_id
-                );
-            }
-        }
-
-        if exclusive_for_vertical {
-            let all_worker_ids: HashSet<_> =
-                streaming_workers_index_by_id.keys().cloned().collect();
-
-            let include_worker_id_set: HashSet<_> = include_worker_ids.iter().cloned().collect();
-            let generated_exclude_worker_ids = all_worker_ids.sub(&include_worker_id_set);
-
-            exclude_worker_ids = exclude_worker_ids
-                .into_iter()
-                .chain(generated_exclude_worker_ids)
-                .unique()
-                .collect();
-        }
-
-        WorkerChanges {
-            include_worker_ids,
-            exclude_worker_ids,
-            target_parallelism,
-            target_parallelism_per_worker,
-        }
-    };
-
-    let all_fragment_ids: HashSet<_> = table_fragments
-        .iter()
-        .flat_map(|table_fragments| table_fragments.fragments.keys().cloned())
-        .collect();
-
-    let target_fragment_ids = match fragments {
-        None => all_fragment_ids.into_iter().collect_vec(),
-        Some(fragment_ids) => {
-            let provide_fragment_ids: HashSet<_> = fragment_ids.into_iter().collect();
-            if provide_fragment_ids
-                .iter()
-                .any(|fragment_id| !all_fragment_ids.contains(fragment_id))
-            {
-                fail!(
-                    "Invalid fragment ids: {:?}",
-                    provide_fragment_ids
-                        .difference(&all_fragment_ids)
-                        .collect_vec()
-                );
-            }
-
-            provide_fragment_ids.into_iter().collect()
-        }
-    };
-
-    let policy = PbPolicy::StableResizePolicy(StableResizePolicy {
-        fragment_worker_changes: target_fragment_ids
-            .iter()
-            .map(|id| (*id as FragmentId, worker_changes.clone()))
-            .collect(),
-    });
-
-    let response = meta_client.get_reschedule_plan(policy, revision).await;
-
-    let GetReschedulePlanResponse {
-        revision,
-        reschedules,
-        success,
-    } = match response {
-        Ok(response) => response,
-        Err(e) => {
-            fail!("Failed to generate plan: {}", e.as_report());
-        }
-    };
-
-    if !success {
-        fail!("Failed to generate plan, current revision is {}", revision);
-    }
-
-    if reschedules.is_empty() {
-        println!(
-            "No reschedule plan generated, no action required, current revision is {}",
-            revision
-        );
-        return Ok(());
-    }
-
-    println!(
-        "Successfully generated plan, current revision is {}",
-        revision
-    );
-
-    if generate {
-        let payload = ReschedulePayload {
-            reschedule_revision: revision,
-            reschedule_plan: reschedules
-                .into_iter()
-                .map(|(fragment_id, reschedule)| (fragment_id, reschedule.into()))
-                .collect(),
-        };
-
-        if let Some(output) = output.as_ref() {
-            println!("Writing plan to file: {}", output);
-            let writer = std::fs::File::create(output)?;
-            serde_yaml::to_writer(writer, &payload)?;
-            println!("Writing plan to file: {} done", output);
-            println!("You can use the `risectl meta reschedule --from {}` command to execute the generated plan", output);
-        } else {
-            println!("Option `--output` is not provided, the result plan will be output to the current command line.");
-            println!("#=========== Payload ==============#");
-            serde_yaml::to_writer(std::io::stdout(), &payload)?;
-            println!("#=========== Payload ==============#");
-        }
-    } else {
-        if !yes {
-            match Confirm::new("Will perform actions on the cluster, are you sure?")
-                .with_default(false)
-                .with_help_message("Use the --generate flag to view the generated plan. Use the --yes or -y option to skip this prompt")
-                .with_placeholder("no")
-                .prompt()
-            {
-                Ok(true) => println!("Processing..."),
-                Ok(false) => {
-                    fail!("Abort.");
-                }
-                Err(_) => {
-                    fail!("Error with questionnaire, try again later");
-                }
-            }
-        }
-
-        let (success, next_revision) =
-            match meta_client.reschedule(reschedules, revision, false).await {
-                Ok(response) => response,
-                Err(e) => {
-                    fail!("Failed to execute plan: {}", e.as_report());
-                }
-            };
-
-        if !success {
-            fail!("Failed to execute plan, current revision is {}", revision);
-        }
-
-        println!(
-            "Successfully executed plan, current revision is {}",
-            next_revision
-        );
-    }
-
-    Ok(())
-}
-
 pub async fn update_schedulability(
     context: &CtlContext,
     workers: Vec<String>,
diff --git a/src/ctl/src/common/context.rs b/src/ctl/src/common/context.rs
index da9f2c6c7b10a..2665a482a1074 100644
--- a/src/ctl/src/common/context.rs
+++ b/src/ctl/src/common/context.rs
@@ -60,8 +60,10 @@ impl CtlContext {
             .cloned()
     }
 
-    pub async fn try_close(mut self) {
+    pub async fn try_close(&self) {
         tracing::info!("clean up context");
-        self.meta_client.take().unwrap().try_unregister().await;
+        if let Some(meta_client) = self.meta_client.get() {
+            meta_client.try_unregister().await;
+        }
     }
 }
diff --git a/src/ctl/src/lib.rs b/src/ctl/src/lib.rs
index 68598e3f9db59..9a82f444079a0 100644
--- a/src/ctl/src/lib.rs
+++ b/src/ctl/src/lib.rs
@@ -20,6 +20,7 @@ use clap::{Args, Parser, Subcommand};
 use cmd_impl::bench::BenchCommands;
 use cmd_impl::hummock::SstDumpArgs;
 use itertools::Itertools;
+use risingwave_common::util::tokio_util::sync::CancellationToken;
 use risingwave_hummock_sdk::HummockEpoch;
 use risingwave_meta::backup_restore::RestoreOpts;
 use risingwave_pb::hummock::rise_ctl_update_compaction_config_request::CompressionAlgorithm;
@@ -209,14 +210,16 @@ enum HummockCommands {
         #[clap(short, long = "level", default_value_t = 1)]
         level: u32,
 
-        #[clap(short, long = "sst-ids")]
+        #[clap(short, long = "sst-ids", value_delimiter = ',')]
         sst_ids: Vec<u64>,
     },
-    /// trigger a full GC for SSTs that is not in version and with timestamp <= now -
-    /// `sst_retention_time_sec`.
+    /// Trigger a full GC for SSTs that is not pinned, with timestamp <= now -
+    /// `sst_retention_time_sec`, and with `prefix` in path.
     TriggerFullGc {
         #[clap(short, long = "sst_retention_time_sec", default_value_t = 259200)]
         sst_retention_time_sec: u64,
+        #[clap(short, long = "prefix", required = false)]
+        prefix: Option<String>,
     },
     /// List pinned versions of each worker.
     ListPinnedVersions {},
@@ -226,7 +229,7 @@ enum HummockCommands {
     ListCompactionGroup,
     /// Update compaction config for compaction groups.
     UpdateCompactionConfig {
-        #[clap(long)]
+        #[clap(long, value_delimiter = ',')]
         compaction_group_ids: Vec<u64>,
         #[clap(long)]
         max_bytes_for_level_base: Option<u64>,
@@ -269,7 +272,7 @@ enum HummockCommands {
     SplitCompactionGroup {
         #[clap(long)]
         compaction_group_id: u64,
-        #[clap(long)]
+        #[clap(long, value_delimiter = ',')]
         table_ids: Vec<u32>,
     },
     /// Pause version checkpoint, which subsequently pauses GC of delta log and SST object.
@@ -362,92 +365,8 @@ enum TableCommands {
     List,
 }
 
-#[derive(clap::Args, Debug, Clone)]
-pub struct ScaleHorizonCommands {
-    /// The worker that needs to be excluded during scheduling, `worker_id` and `worker_host:worker_port` are both
-    /// supported
-    #[clap(
-        long,
-        value_delimiter = ',',
-        value_name = "worker_id or worker_host:worker_port, ..."
-    )]
-    exclude_workers: Option<Vec<String>>,
-
-    /// The worker that needs to be included during scheduling, `worker_id` and `worker_host:worker_port` are both
-    /// supported
-    #[clap(
-        long,
-        value_delimiter = ',',
-        value_name = "all or worker_id or worker_host:worker_port, ..."
-    )]
-    include_workers: Option<Vec<String>>,
-
-    /// The target parallelism, currently, it is used to limit the target parallelism and only
-    /// takes effect when the actual parallelism exceeds this value. Can be used in conjunction
-    /// with `exclude/include_workers`.
-    #[clap(long)]
-    target_parallelism: Option<u32>,
-
-    #[command(flatten)]
-    common: ScaleCommon,
-}
-
-#[derive(clap::Args, Debug, Clone)]
-pub struct ScaleCommon {
-    /// Will generate a plan supported by the `reschedule` command and save it to the provided path
-    /// by the `--output`.
-    #[clap(long, default_value_t = false)]
-    generate: bool,
-
-    /// The output file to write the generated plan to, standard output by default
-    #[clap(long)]
-    output: Option<String>,
-
-    /// Automatic yes to prompts
-    #[clap(short = 'y', long, default_value_t = false)]
-    yes: bool,
-
-    /// Specify the fragment ids that need to be scheduled.
-    /// empty by default, which means all fragments will be scheduled
-    #[clap(long, value_delimiter = ',')]
-    fragments: Option<Vec<u32>>,
-}
-
-#[derive(clap::Args, Debug, Clone)]
-pub struct ScaleVerticalCommands {
-    #[command(flatten)]
-    common: ScaleCommon,
-
-    /// The worker that needs to be scheduled, `worker_id` and `worker_host:worker_port` are both
-    /// supported
-    #[clap(
-        long,
-        required = true,
-        value_delimiter = ',',
-        value_name = "all or worker_id or worker_host:worker_port, ..."
-    )]
-    workers: Option<Vec<String>>,
-
-    /// The target parallelism per worker, requires `workers` to be set.
-    #[clap(long, required = true)]
-    target_parallelism_per_worker: Option<u32>,
-
-    /// It will exclude all other workers to maintain the target parallelism only for the target workers.
-    #[clap(long, default_value_t = false)]
-    exclusive: bool,
-}
-
 #[derive(Subcommand, Debug)]
 enum ScaleCommands {
-    /// Scale the compute nodes horizontally, alias of `horizon`
-    Resize(ScaleHorizonCommands),
-
-    /// Scale the compute nodes horizontally
-    Horizon(ScaleHorizonCommands),
-
-    /// Scale the compute nodes vertically
-    Vertical(ScaleVerticalCommands),
-
     /// Mark a compute node as unschedulable
     #[clap(verbatim_doc_comment)]
     Cordon {
@@ -484,18 +403,18 @@ enum MetaCommands {
     ClusterInfo,
     /// get source split info
     SourceSplitInfo,
-    /// Reschedule the parallel unit in the stream graph
+    /// Reschedule the actors in the stream graph
     ///
-    /// The format is `fragment_id-[removed]+[added]`
-    /// You can provide either `removed` only or `added` only, but `removed` should be preceded by
+    /// The format is `fragment_id-[worker_id:count]+[worker_id:count]`
+    /// You can provide either decreased `worker_ids` only or increased only, but decreased should be preceded by
     /// `added` when both are provided.
     ///
-    /// For example, for plan `100-[1,2,3]+[4,5]` the follow request will be generated:
+    /// For example, for plan `100-[1:1]+[4:1]` the follow request will be generated:
     /// ```text
     /// {
-    ///     100: Reschedule {
-    ///         added_parallel_units: [4,5],
-    ///         removed_parallel_units: [1,2,3],
+    ///     100: WorkerReschedule {
+    ///         increased_actor_count: { 1: 1 },
+    ///         decreased_actor_count: { 4: 1 },
     ///     }
     /// }
     /// ```
@@ -530,12 +449,15 @@ enum MetaCommands {
         opts: RestoreOpts,
     },
     /// delete meta snapshots
-    DeleteMetaSnapshots { snapshot_ids: Vec<u64> },
+    DeleteMetaSnapshots {
+        #[clap(long, value_delimiter = ',')]
+        snapshot_ids: Vec<u64>,
+    },
 
     /// List all existing connections in the catalog
     ListConnections,
 
-    /// List fragment to parallel units mapping for serving
+    /// List fragment mapping for serving
     ListServingFragmentMapping,
 
     /// Unregister workers from the cluster
@@ -623,22 +545,33 @@ pub enum ProfileCommands {
 }
 
 /// Start `risectl` with the given options.
+/// Cancel the operation when the given `shutdown` token triggers.
 /// Log and abort the process if any error occurs.
 ///
 /// Note: use [`start_fallible`] if you want to call functionalities of `risectl`
 /// in an embedded manner.
-pub async fn start(opts: CliOpts) {
-    if let Err(e) = start_fallible(opts).await {
-        eprintln!("Error: {:#?}", e.as_report()); // pretty with backtrace
-        std::process::exit(1);
+pub async fn start(opts: CliOpts, shutdown: CancellationToken) {
+    let context = CtlContext::default();
+
+    tokio::select! {
+        _ = shutdown.cancelled() => {
+            // Shutdown requested, clean up the context and return.
+            context.try_close().await;
+        }
+
+        result = start_fallible(opts, &context) => {
+            if let Err(e) = result {
+                eprintln!("Error: {:#?}", e.as_report()); // pretty with backtrace
+                std::process::exit(1);
+            }
+        }
     }
 }
 
 /// Start `risectl` with the given options.
 /// Return `Err` if any error occurs.
-pub async fn start_fallible(opts: CliOpts) -> Result<()> {
-    let context = CtlContext::default();
-    let result = start_impl(opts, &context).await;
+pub async fn start_fallible(opts: CliOpts, context: &CtlContext) -> Result<()> {
+    let result = start_impl(opts, context).await;
     context.try_close().await;
     result
 }
@@ -698,7 +631,8 @@ async fn start_impl(opts: CliOpts, context: &CtlContext) -> Result<()> {
         }
         Commands::Hummock(HummockCommands::TriggerFullGc {
             sst_retention_time_sec,
-        }) => cmd_impl::hummock::trigger_full_gc(context, sst_retention_time_sec).await?,
+            prefix,
+        }) => cmd_impl::hummock::trigger_full_gc(context, sst_retention_time_sec, prefix).await?,
         Commands::Hummock(HummockCommands::ListPinnedVersions {}) => {
             list_pinned_versions(context).await?
         }
@@ -942,13 +876,6 @@ async fn start_impl(opts: CliOpts, context: &CtlContext) -> Result<()> {
         Commands::Profile(ProfileCommands::Heap { dir }) => {
             cmd_impl::profile::heap_profile(context, dir).await?
         }
-        Commands::Scale(ScaleCommands::Horizon(resize))
-        | Commands::Scale(ScaleCommands::Resize(resize)) => {
-            cmd_impl::scale::resize(context, resize.into()).await?
-        }
-        Commands::Scale(ScaleCommands::Vertical(resize)) => {
-            cmd_impl::scale::resize(context, resize.into()).await?
-        }
         Commands::Scale(ScaleCommands::Cordon { workers }) => {
             cmd_impl::scale::update_schedulability(context, workers, Schedulability::Unschedulable)
                 .await?
diff --git a/src/error/src/tonic.rs b/src/error/src/tonic.rs
index 11cbb71063652..4e3476c460fd6 100644
--- a/src/error/src/tonic.rs
+++ b/src/error/src/tonic.rs
@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+pub mod extra;
+
 use std::borrow::Cow;
 use std::error::Error;
 use std::sync::Arc;
@@ -24,6 +26,7 @@ use tonic::metadata::{MetadataMap, MetadataValue};
 const ERROR_KEY: &str = "risingwave-error-bin";
 
 /// The service name that the error is from. Used to provide better error message.
+// TODO: also make it a field of `Extra`?
 type ServiceName = Cow<'static, str>;
 
 /// The error produced by the gRPC server and sent to the client on the wire.
@@ -31,6 +34,7 @@ type ServiceName = Cow<'static, str>;
 struct ServerError {
     error: serde_error::Error,
     service_name: Option<ServiceName>,
+    extra: extra::Extra,
 }
 
 impl std::fmt::Display for ServerError {
@@ -43,6 +47,10 @@ impl std::error::Error for ServerError {
     fn source(&self) -> Option<&(dyn Error + 'static)> {
         self.error.source()
     }
+
+    fn provide<'a>(&'a self, request: &mut std::error::Request<'a>) {
+        self.extra.provide(request);
+    }
 }
 
 fn to_status<T>(error: &T, code: tonic::Code, service_name: Option<ServiceName>) -> tonic::Status
@@ -55,6 +63,7 @@ where
     let source = ServerError {
         error: serde_error::Error::new(error),
         service_name,
+        extra: extra::Extra::new(error),
     };
     let serialized = bincode::serialize(&source).unwrap();
 
@@ -204,6 +213,13 @@ impl std::error::Error for TonicStatusWrapper {
         // Delegate to `self.inner` as if we're transparent.
         self.inner.source()
     }
+
+    fn provide<'a>(&'a self, request: &mut std::error::Request<'a>) {
+        // The source error, typically a `ServerError`, may provide additional information through `extra`.
+        if let Some(source) = self.source() {
+            source.provide(request);
+        }
+    }
 }
 
 #[cfg(test)]
diff --git a/src/error/src/tonic/extra.rs b/src/error/src/tonic/extra.rs
new file mode 100644
index 0000000000000..dbf6b80e29124
--- /dev/null
+++ b/src/error/src/tonic/extra.rs
@@ -0,0 +1,50 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use serde::{Deserialize, Serialize};
+
+/// The score of the error.
+///
+/// Currently, it's used to identify the root cause of streaming pipeline failures, i.e., which actor
+/// led to the failure.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
+pub struct Score(pub i32);
+
+/// Extra fields in errors that can be passed through the gRPC boundary.
+///
+/// - Field being set to `None` means it is not available.
+/// - To add a new field, also update the `provide` method.
+#[derive(Debug, Default, Clone, Serialize, Deserialize)]
+pub(super) struct Extra {
+    pub score: Option<Score>,
+}
+
+impl Extra {
+    /// Create a new [`Extra`] by [requesting](std::error::request_ref) each field from the given error.
+    pub fn new<T>(error: &T) -> Self
+    where
+        T: ?Sized + std::error::Error,
+    {
+        Self {
+            score: std::error::request_value(error),
+        }
+    }
+
+    /// Provide each field to the given [request](std::error::Request).
+    pub fn provide<'a>(&'a self, request: &mut std::error::Request<'a>) {
+        if let Some(score) = self.score {
+            request.provide_value(score);
+        }
+    }
+}
diff --git a/src/expr/core/src/aggregate/def.rs b/src/expr/core/src/aggregate/def.rs
index 5ca07d0153196..b050f8039e1c6 100644
--- a/src/expr/core/src/aggregate/def.rs
+++ b/src/expr/core/src/aggregate/def.rs
@@ -14,17 +14,20 @@
 
 //! Aggregation function definitions.
 
+use std::fmt::Display;
 use std::iter::Peekable;
+use std::str::FromStr;
 use std::sync::Arc;
 
+use anyhow::Context;
+use enum_as_inner::EnumAsInner;
 use itertools::Itertools;
-use parse_display::{Display, FromStr};
 use risingwave_common::bail;
 use risingwave_common::types::{DataType, Datum};
 use risingwave_common::util::sort_util::{ColumnOrder, OrderType};
 use risingwave_common::util::value_encoding::DatumFromProtoExt;
-use risingwave_pb::expr::agg_call::PbType;
-use risingwave_pb::expr::{PbAggCall, PbInputRef, PbUserDefinedFunctionMetadata};
+pub use risingwave_pb::expr::agg_call::PbType as PbAggKind;
+use risingwave_pb::expr::{PbAggCall, PbExprNode, PbInputRef, PbUserDefinedFunctionMetadata};
 
 use crate::expr::{
     build_from_prost, BoxedExpression, ExpectExt, Expression, LiteralExpression, Token,
@@ -40,8 +43,10 @@ use crate::Result;
 pub struct AggCall {
     /// Aggregation kind for constructing agg state.
     pub kind: AggKind,
+
     /// Arguments of aggregation function input.
     pub args: AggArgs,
+
     /// The return type of aggregation function.
     pub return_type: DataType,
 
@@ -56,14 +61,15 @@ pub struct AggCall {
 
     /// Constant arguments.
     pub direct_args: Vec<LiteralExpression>,
-
-    /// Additional metadata for user defined functions.
-    pub user_defined: Option<PbUserDefinedFunctionMetadata>,
 }
 
 impl AggCall {
     pub fn from_protobuf(agg_call: &PbAggCall) -> Result<Self> {
-        let agg_kind = AggKind::from_protobuf(agg_call.get_type()?)?;
+        let agg_kind = AggKind::from_protobuf(
+            agg_call.get_type()?,
+            agg_call.udf.as_ref(),
+            agg_call.scalar.as_ref(),
+        )?;
         let args = AggArgs::from_protobuf(agg_call.get_args())?;
         let column_orders = agg_call
             .get_order_by()
@@ -97,7 +103,6 @@ impl AggCall {
             filter,
             distinct: agg_call.distinct,
             direct_args,
-            user_defined: agg_call.udf.clone(),
         })
     }
 
@@ -155,7 +160,7 @@ impl<Iter: Iterator<Item = Token>> Parser<Iter> {
         self.tokens.next(); // Consume the RParen
 
         AggCall {
-            kind: AggKind::from_protobuf(func).unwrap(),
+            kind: AggKind::from_protobuf(func, None, None).unwrap(),
             args: AggArgs {
                 data_types: children.iter().map(|(_, ty)| ty.clone()).collect(),
                 val_indices: children.iter().map(|(idx, _)| *idx).collect(),
@@ -165,7 +170,6 @@ impl<Iter: Iterator<Item = Token>> Parser<Iter> {
             filter: None,
             distinct,
             direct_args: Vec::new(),
-            user_defined: None,
         }
     }
 
@@ -186,10 +190,10 @@ impl<Iter: Iterator<Item = Token>> Parser<Iter> {
         (idx, ty)
     }
 
-    fn parse_function(&mut self) -> PbType {
+    fn parse_function(&mut self) -> PbAggKind {
         match self.tokens.next().expect("Unexpected end of input") {
             Token::Literal(name) => {
-                PbType::from_str_name(&name.to_uppercase()).expect_str("function", &name)
+                PbAggKind::from_str_name(&name.to_uppercase()).expect_str("function", &name)
             }
             t => panic!("Expected a Literal, got {t:?}"),
         }
@@ -210,191 +214,170 @@ impl<Iter: Iterator<Item = Token>> Parser<Iter> {
     }
 }
 
-/// Kind of aggregation function
-#[derive(Debug, Display, FromStr, Copy, Clone, PartialEq, Eq, Hash)]
-#[display(style = "snake_case")]
+/// Aggregate function kind.
+#[derive(Debug, Clone, PartialEq, Eq, Hash, EnumAsInner)]
 pub enum AggKind {
-    BitAnd,
-    BitOr,
-    BitXor,
-    BoolAnd,
-    BoolOr,
-    Min,
-    Max,
-    Sum,
-    Sum0,
-    Count,
-    Avg,
-    StringAgg,
-    ApproxCountDistinct,
-    ArrayAgg,
-    JsonbAgg,
-    JsonbObjectAgg,
-    FirstValue,
-    LastValue,
-    VarPop,
-    VarSamp,
-    StddevPop,
-    StddevSamp,
-    PercentileCont,
-    PercentileDisc,
-    Mode,
-    Grouping,
-
-    /// Return last seen one of the input values.
-    InternalLastSeenValue,
-    UserDefined,
+    /// Built-in aggregate function.
+    ///
+    /// The associated value should not be `UserDefined` or `WrapScalar`.
+    Builtin(PbAggKind),
+
+    /// User defined aggregate function.
+    UserDefined(PbUserDefinedFunctionMetadata),
+
+    /// Wrap a scalar function that takes a list as input as an aggregation function.
+    WrapScalar(PbExprNode),
+}
+
+impl Display for AggKind {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Self::Builtin(kind) => write!(f, "{}", kind.as_str_name().to_lowercase()),
+            Self::UserDefined(_) => write!(f, "udaf"),
+            Self::WrapScalar(_) => write!(f, "wrap_scalar"),
+        }
+    }
+}
+
+impl FromStr for AggKind {
+    type Err = ();
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        let kind = PbAggKind::from_str(s)?;
+        Ok(AggKind::Builtin(kind))
+    }
+}
+
+impl From<PbAggKind> for AggKind {
+    fn from(pb: PbAggKind) -> Self {
+        assert!(!matches!(
+            pb,
+            PbAggKind::Unspecified | PbAggKind::UserDefined | PbAggKind::WrapScalar
+        ));
+        AggKind::Builtin(pb)
+    }
 }
 
 impl AggKind {
-    pub fn from_protobuf(pb_type: PbType) -> Result<Self> {
+    pub fn from_protobuf(
+        pb_type: PbAggKind,
+        user_defined: Option<&PbUserDefinedFunctionMetadata>,
+        scalar: Option<&PbExprNode>,
+    ) -> Result<Self> {
         match pb_type {
-            PbType::BitAnd => Ok(AggKind::BitAnd),
-            PbType::BitOr => Ok(AggKind::BitOr),
-            PbType::BitXor => Ok(AggKind::BitXor),
-            PbType::BoolAnd => Ok(AggKind::BoolAnd),
-            PbType::BoolOr => Ok(AggKind::BoolOr),
-            PbType::Min => Ok(AggKind::Min),
-            PbType::Max => Ok(AggKind::Max),
-            PbType::Sum => Ok(AggKind::Sum),
-            PbType::Sum0 => Ok(AggKind::Sum0),
-            PbType::Avg => Ok(AggKind::Avg),
-            PbType::Count => Ok(AggKind::Count),
-            PbType::StringAgg => Ok(AggKind::StringAgg),
-            PbType::ApproxCountDistinct => Ok(AggKind::ApproxCountDistinct),
-            PbType::ArrayAgg => Ok(AggKind::ArrayAgg),
-            PbType::JsonbAgg => Ok(AggKind::JsonbAgg),
-            PbType::JsonbObjectAgg => Ok(AggKind::JsonbObjectAgg),
-            PbType::FirstValue => Ok(AggKind::FirstValue),
-            PbType::LastValue => Ok(AggKind::LastValue),
-            PbType::StddevPop => Ok(AggKind::StddevPop),
-            PbType::StddevSamp => Ok(AggKind::StddevSamp),
-            PbType::VarPop => Ok(AggKind::VarPop),
-            PbType::VarSamp => Ok(AggKind::VarSamp),
-            PbType::PercentileCont => Ok(AggKind::PercentileCont),
-            PbType::PercentileDisc => Ok(AggKind::PercentileDisc),
-            PbType::Mode => Ok(AggKind::Mode),
-            PbType::Grouping => Ok(AggKind::Grouping),
-            PbType::InternalLastSeenValue => Ok(AggKind::InternalLastSeenValue),
-            PbType::UserDefined => Ok(AggKind::UserDefined),
-            PbType::Unspecified => bail!("Unrecognized agg."),
+            PbAggKind::UserDefined => {
+                let user_defined = user_defined.context("expect user defined")?;
+                Ok(AggKind::UserDefined(user_defined.clone()))
+            }
+            PbAggKind::WrapScalar => {
+                let scalar = scalar.context("expect scalar")?;
+                Ok(AggKind::WrapScalar(scalar.clone()))
+            }
+            PbAggKind::Unspecified => bail!("Unrecognized agg."),
+            _ => Ok(AggKind::Builtin(pb_type)),
         }
     }
 
-    pub fn to_protobuf(self) -> PbType {
+    pub fn to_protobuf(&self) -> PbAggKind {
         match self {
-            Self::BitAnd => PbType::BitAnd,
-            Self::BitOr => PbType::BitOr,
-            Self::BitXor => PbType::BitXor,
-            Self::BoolAnd => PbType::BoolAnd,
-            Self::BoolOr => PbType::BoolOr,
-            Self::Min => PbType::Min,
-            Self::Max => PbType::Max,
-            Self::Sum => PbType::Sum,
-            Self::Sum0 => PbType::Sum0,
-            Self::Avg => PbType::Avg,
-            Self::Count => PbType::Count,
-            Self::StringAgg => PbType::StringAgg,
-            Self::ApproxCountDistinct => PbType::ApproxCountDistinct,
-            Self::ArrayAgg => PbType::ArrayAgg,
-            Self::JsonbAgg => PbType::JsonbAgg,
-            Self::JsonbObjectAgg => PbType::JsonbObjectAgg,
-            Self::FirstValue => PbType::FirstValue,
-            Self::LastValue => PbType::LastValue,
-            Self::StddevPop => PbType::StddevPop,
-            Self::StddevSamp => PbType::StddevSamp,
-            Self::VarPop => PbType::VarPop,
-            Self::VarSamp => PbType::VarSamp,
-            Self::PercentileCont => PbType::PercentileCont,
-            Self::PercentileDisc => PbType::PercentileDisc,
-            Self::Mode => PbType::Mode,
-            Self::Grouping => PbType::Grouping,
-            Self::InternalLastSeenValue => PbType::InternalLastSeenValue,
-            Self::UserDefined => PbType::UserDefined,
+            Self::Builtin(pb) => *pb,
+            Self::UserDefined(_) => PbAggKind::UserDefined,
+            Self::WrapScalar(_) => PbAggKind::WrapScalar,
         }
     }
 }
 
-/// Macros to generate match arms for [`AggKind`](crate::aggregate::AggKind).
+/// Macros to generate match arms for [`AggKind`](AggKind).
 /// IMPORTANT: These macros must be carefully maintained especially when adding new
-/// [`AggKind`](crate::aggregate::AggKind) variants.
+/// [`AggKind`](AggKind) variants.
 pub mod agg_kinds {
-    /// [`AggKind`](crate::aggregate::AggKind)s that are currently not supported in streaming mode.
+    /// [`AggKind`](super::AggKind)s that are currently not supported in streaming mode.
     #[macro_export]
     macro_rules! unimplemented_in_stream {
         () => {
-            AggKind::PercentileCont | AggKind::PercentileDisc | AggKind::Mode
+            AggKind::Builtin(
+                PbAggKind::PercentileCont | PbAggKind::PercentileDisc | PbAggKind::Mode,
+            )
         };
     }
     pub use unimplemented_in_stream;
 
-    /// [`AggKind`](crate::aggregate::AggKind)s that should've been rewritten to other kinds. These kinds
+    /// [`AggKind`](super::AggKind)s that should've been rewritten to other kinds. These kinds
     /// should not appear when generating physical plan nodes.
     #[macro_export]
     macro_rules! rewritten {
         () => {
-            AggKind::Avg
-                | AggKind::StddevPop
-                | AggKind::StddevSamp
-                | AggKind::VarPop
-                | AggKind::VarSamp
-                | AggKind::Grouping
+            AggKind::Builtin(
+                PbAggKind::Avg
+                    | PbAggKind::StddevPop
+                    | PbAggKind::StddevSamp
+                    | PbAggKind::VarPop
+                    | PbAggKind::VarSamp
+                    | PbAggKind::Grouping
+                    // ApproxPercentile always uses custom agg executors,
+                    // rather than an aggregation operator
+                    | PbAggKind::ApproxPercentile
+            )
         };
     }
     pub use rewritten;
 
-    /// [`AggKind`](crate::aggregate::AggKind)s of which the aggregate results are not affected by the
+    /// [`AggKind`](super::AggKind)s of which the aggregate results are not affected by the
     /// user given ORDER BY clause.
     #[macro_export]
     macro_rules! result_unaffected_by_order_by {
         () => {
-            AggKind::BitAnd
-                | AggKind::BitOr
-                | AggKind::BitXor // XOR is commutative and associative
-                | AggKind::BoolAnd
-                | AggKind::BoolOr
-                | AggKind::Min
-                | AggKind::Max
-                | AggKind::Sum
-                | AggKind::Sum0
-                | AggKind::Count
-                | AggKind::Avg
-                | AggKind::ApproxCountDistinct
-                | AggKind::VarPop
-                | AggKind::VarSamp
-                | AggKind::StddevPop
-                | AggKind::StddevSamp
+            AggKind::Builtin(PbAggKind::BitAnd
+                | PbAggKind::BitOr
+                | PbAggKind::BitXor // XOR is commutative and associative
+                | PbAggKind::BoolAnd
+                | PbAggKind::BoolOr
+                | PbAggKind::Min
+                | PbAggKind::Max
+                | PbAggKind::Sum
+                | PbAggKind::Sum0
+                | PbAggKind::Count
+                | PbAggKind::Avg
+                | PbAggKind::ApproxCountDistinct
+                | PbAggKind::VarPop
+                | PbAggKind::VarSamp
+                | PbAggKind::StddevPop
+                | PbAggKind::StddevSamp)
         };
     }
     pub use result_unaffected_by_order_by;
 
-    /// [`AggKind`](crate::aggregate::AggKind)s that must be called with ORDER BY clause. These are
+    /// [`AggKind`](super::AggKind)s that must be called with ORDER BY clause. These are
     /// slightly different from variants not in [`result_unaffected_by_order_by`], in that
     /// variants returned by this macro should be banned while the others should just be warned.
     #[macro_export]
     macro_rules! must_have_order_by {
         () => {
-            AggKind::FirstValue
-                | AggKind::LastValue
-                | AggKind::PercentileCont
-                | AggKind::PercentileDisc
-                | AggKind::Mode
+            AggKind::Builtin(
+                PbAggKind::FirstValue
+                    | PbAggKind::LastValue
+                    | PbAggKind::PercentileCont
+                    | PbAggKind::PercentileDisc
+                    | PbAggKind::Mode,
+            )
         };
     }
     pub use must_have_order_by;
 
-    /// [`AggKind`](crate::aggregate::AggKind)s of which the aggregate results are not affected by the
+    /// [`AggKind`](super::AggKind)s of which the aggregate results are not affected by the
     /// user given DISTINCT keyword.
     #[macro_export]
     macro_rules! result_unaffected_by_distinct {
         () => {
-            AggKind::BitAnd
-                | AggKind::BitOr
-                | AggKind::BoolAnd
-                | AggKind::BoolOr
-                | AggKind::Min
-                | AggKind::Max
-                | AggKind::ApproxCountDistinct
+            AggKind::Builtin(
+                PbAggKind::BitAnd
+                    | PbAggKind::BitOr
+                    | PbAggKind::BoolAnd
+                    | PbAggKind::BoolOr
+                    | PbAggKind::Min
+                    | PbAggKind::Max
+                    | PbAggKind::ApproxCountDistinct,
+            )
         };
     }
     pub use result_unaffected_by_distinct;
@@ -403,53 +386,58 @@ pub mod agg_kinds {
     #[macro_export]
     macro_rules! simply_cannot_two_phase {
         () => {
-            AggKind::StringAgg
-                | AggKind::ApproxCountDistinct
-                | AggKind::ArrayAgg
-                | AggKind::JsonbAgg
-                | AggKind::JsonbObjectAgg
-                | AggKind::FirstValue
-                | AggKind::LastValue
-                | AggKind::PercentileCont
-                | AggKind::PercentileDisc
-                | AggKind::Mode
-                // FIXME(wrj): move `BoolAnd` and `BoolOr` out
-                //  after we support general merge in stateless_simple_agg
-                | AggKind::BoolAnd
-                | AggKind::BoolOr
-                | AggKind::BitAnd
-                | AggKind::BitOr
-                | AggKind::UserDefined
+            AggKind::Builtin(
+                PbAggKind::StringAgg
+                    | PbAggKind::ApproxCountDistinct
+                    | PbAggKind::ArrayAgg
+                    | PbAggKind::JsonbAgg
+                    | PbAggKind::JsonbObjectAgg
+                    | PbAggKind::FirstValue
+                    | PbAggKind::LastValue
+                    | PbAggKind::PercentileCont
+                    | PbAggKind::PercentileDisc
+                    | PbAggKind::Mode
+                    // FIXME(wrj): move `BoolAnd` and `BoolOr` out
+                    //  after we support general merge in stateless_simple_agg
+                    | PbAggKind::BoolAnd
+                    | PbAggKind::BoolOr
+                    | PbAggKind::BitAnd
+                    | PbAggKind::BitOr
+            )
+            | AggKind::UserDefined(_)
+            | AggKind::WrapScalar(_)
         };
     }
     pub use simply_cannot_two_phase;
 
-    /// [`AggKind`](crate::aggregate::AggKind)s that are implemented with a single value state (so-called
+    /// [`AggKind`](super::AggKind)s that are implemented with a single value state (so-called
     /// stateless).
     #[macro_export]
     macro_rules! single_value_state {
         () => {
-            AggKind::Sum
-                | AggKind::Sum0
-                | AggKind::Count
-                | AggKind::BitAnd
-                | AggKind::BitOr
-                | AggKind::BitXor
-                | AggKind::BoolAnd
-                | AggKind::BoolOr
-                | AggKind::ApproxCountDistinct
-                | AggKind::InternalLastSeenValue
-                | AggKind::UserDefined
+            AggKind::Builtin(
+                PbAggKind::Sum
+                    | PbAggKind::Sum0
+                    | PbAggKind::Count
+                    | PbAggKind::BitAnd
+                    | PbAggKind::BitOr
+                    | PbAggKind::BitXor
+                    | PbAggKind::BoolAnd
+                    | PbAggKind::BoolOr
+                    | PbAggKind::ApproxCountDistinct
+                    | PbAggKind::InternalLastSeenValue
+                    | PbAggKind::ApproxPercentile,
+            ) | AggKind::UserDefined(_)
         };
     }
     pub use single_value_state;
 
-    /// [`AggKind`](crate::aggregate::AggKind)s that are implemented with a single value state (so-called
+    /// [`AggKind`](super::AggKind)s that are implemented with a single value state (so-called
     /// stateless) iff the input is append-only.
     #[macro_export]
     macro_rules! single_value_state_iff_in_append_only {
         () => {
-            AggKind::Max | AggKind::Min
+            AggKind::Builtin(PbAggKind::Max | PbAggKind::Min)
         };
     }
     pub use single_value_state_iff_in_append_only;
@@ -458,7 +446,12 @@ pub mod agg_kinds {
     #[macro_export]
     macro_rules! ordered_set {
         () => {
-            AggKind::PercentileCont | AggKind::PercentileDisc | AggKind::Mode
+            AggKind::Builtin(
+                PbAggKind::PercentileCont
+                    | PbAggKind::PercentileDisc
+                    | PbAggKind::Mode
+                    | PbAggKind::ApproxPercentile,
+            )
         };
     }
     pub use ordered_set;
@@ -466,16 +459,24 @@ pub mod agg_kinds {
 
 impl AggKind {
     /// Get the total phase agg kind from the partial phase agg kind.
-    pub fn partial_to_total(self) -> Option<Self> {
+    pub fn partial_to_total(&self) -> Option<Self> {
         match self {
-            AggKind::BitXor
-            | AggKind::Min
-            | AggKind::Max
-            | AggKind::Sum
-            | AggKind::InternalLastSeenValue => Some(self),
-            AggKind::Sum0 | AggKind::Count => Some(AggKind::Sum0),
+            AggKind::Builtin(
+                PbAggKind::BitXor
+                | PbAggKind::Min
+                | PbAggKind::Max
+                | PbAggKind::Sum
+                | PbAggKind::InternalLastSeenValue,
+            ) => Some(self.clone()),
+            AggKind::Builtin(PbAggKind::Sum0 | PbAggKind::Count) => {
+                Some(Self::Builtin(PbAggKind::Sum0))
+            }
             agg_kinds::simply_cannot_two_phase!() => None,
             agg_kinds::rewritten!() => None,
+            // invalid variants
+            AggKind::Builtin(
+                PbAggKind::Unspecified | PbAggKind::UserDefined | PbAggKind::WrapScalar,
+            ) => None,
         }
     }
 }
diff --git a/src/expr/core/src/aggregate/mod.rs b/src/expr/core/src/aggregate/mod.rs
index 2a1119d6fe301..695aad482343d 100644
--- a/src/expr/core/src/aggregate/mod.rs
+++ b/src/expr/core/src/aggregate/mod.rs
@@ -22,11 +22,13 @@ use risingwave_common::array::StreamChunk;
 use risingwave_common::types::{DataType, Datum};
 use risingwave_common_estimate_size::EstimateSize;
 
+use crate::expr::build_from_prost;
 use crate::sig::FuncBuilder;
 use crate::{ExprError, Result};
 
 // aggregate definition
 mod def;
+mod scalar_wrapper;
 // user defined aggregate function
 mod user_defined;
 
@@ -143,16 +145,27 @@ pub fn build_retractable(agg: &AggCall) -> Result<BoxedAggregateFunction> {
 /// NOTE: This function ignores argument indices, `column_orders`, `filter` and `distinct` in
 /// `AggCall`. Such operations should be done in batch or streaming executors.
 pub fn build(agg: &AggCall, prefer_append_only: bool) -> Result<BoxedAggregateFunction> {
-    if agg.kind == AggKind::UserDefined {
-        return user_defined::new_user_defined(agg);
-    }
+    // handle special kinds
+    let kind = match &agg.kind {
+        AggKind::UserDefined(udf) => {
+            return user_defined::new_user_defined(&agg.return_type, udf);
+        }
+        AggKind::WrapScalar(scalar) => {
+            return Ok(Box::new(scalar_wrapper::ScalarWrapper::new(
+                agg.args.arg_types()[0].clone(),
+                build_from_prost(scalar)?,
+            )));
+        }
+        AggKind::Builtin(kind) => kind,
+    };
 
+    // find the signature for builtin aggregation
     let sig = crate::sig::FUNCTION_REGISTRY
-        .get(agg.kind, agg.args.arg_types(), &agg.return_type)
+        .get(*kind, agg.args.arg_types(), &agg.return_type)
         .ok_or_else(|| {
             ExprError::UnsupportedFunction(format!(
                 "{}({}) -> {}",
-                agg.kind.to_protobuf().as_str_name().to_ascii_lowercase(),
+                kind.as_str_name().to_ascii_lowercase(),
                 agg.args.arg_types().iter().format(", "),
                 agg.return_type,
             ))
diff --git a/src/expr/core/src/aggregate/scalar_wrapper.rs b/src/expr/core/src/aggregate/scalar_wrapper.rs
new file mode 100644
index 0000000000000..7cfa0106baa90
--- /dev/null
+++ b/src/expr/core/src/aggregate/scalar_wrapper.rs
@@ -0,0 +1,111 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use risingwave_common::array::{Array, ArrayBuilderImpl, DataChunk, ListArray};
+use risingwave_common::types::ListValue;
+
+use super::*;
+use crate::expr::{BoxedExpression, Expression};
+
+/// Wraps a scalar function that takes a list as input as an aggregate function.
+#[derive(Debug)]
+pub struct ScalarWrapper {
+    arg_type: DataType,
+    scalar: BoxedExpression,
+}
+
+impl ScalarWrapper {
+    /// Creates a new scalar wrapper.
+    pub fn new(arg_type: DataType, scalar: BoxedExpression) -> Self {
+        Self { arg_type, scalar }
+    }
+}
+
+#[async_trait::async_trait]
+impl AggregateFunction for ScalarWrapper {
+    fn return_type(&self) -> DataType {
+        self.scalar.return_type()
+    }
+
+    /// Creates an initial state of the aggregate function.
+    fn create_state(&self) -> Result<AggregateState> {
+        Ok(AggregateState::Any(Box::new(State(
+            self.arg_type.create_array_builder(0),
+        ))))
+    }
+
+    /// Update the state with multiple rows.
+    ///
+    /// All rows in the input chunk must be Insert.
+    async fn update(&self, state: &mut AggregateState, input: &StreamChunk) -> Result<()> {
+        let state = &mut state.downcast_mut::<State>().0;
+        let column = input.column_at(0);
+        for i in input.visibility().iter_ones() {
+            state.append(column.datum_at(i));
+        }
+        Ok(())
+    }
+
+    /// Update the state with a range of rows.
+    async fn update_range(
+        &self,
+        state: &mut AggregateState,
+        input: &StreamChunk,
+        range: Range<usize>,
+    ) -> Result<()> {
+        let state = &mut state.downcast_mut::<State>().0;
+        let column = input.column_at(0);
+        for i in input.visibility().iter_ones() {
+            if i < range.start {
+                continue;
+            } else if i >= range.end {
+                break;
+            }
+            state.append(column.datum_at(i));
+        }
+        Ok(())
+    }
+
+    /// Get aggregate result from the state.
+    async fn get_result(&self, state: &AggregateState) -> Result<Datum> {
+        let state = &state.downcast_ref::<State>().0;
+        // XXX: can we avoid cloning here?
+        let list = ListValue::new(state.clone().finish());
+        let chunk = DataChunk::new(vec![ListArray::from_iter([list]).into_ref()], 1);
+        let output = self.scalar.eval(&chunk).await?;
+        Ok(output.to_datum())
+    }
+
+    /// Encode the state into a datum that can be stored in state table.
+    fn encode_state(&self, _state: &AggregateState) -> Result<Datum> {
+        panic!("should not store state in state table")
+    }
+
+    /// Decode the state from a datum in state table.
+    fn decode_state(&self, _datum: Datum) -> Result<AggregateState> {
+        panic!("should not store state in state table")
+    }
+}
+
+/// The state is an array builder
+#[derive(Debug)]
+struct State(ArrayBuilderImpl);
+
+impl EstimateSize for State {
+    fn estimated_heap_size(&self) -> usize {
+        self.0.estimated_heap_size()
+    }
+}
+
+impl AggStateDyn for State {}
diff --git a/src/expr/core/src/aggregate/user_defined.rs b/src/expr/core/src/aggregate/user_defined.rs
index 067063c22f4a2..a3897896a3a67 100644
--- a/src/expr/core/src/aggregate/user_defined.rs
+++ b/src/expr/core/src/aggregate/user_defined.rs
@@ -19,7 +19,8 @@ use arrow_array::ArrayRef;
 use arrow_schema::{Field, Fields, Schema, SchemaRef};
 use risingwave_common::array::arrow::{FromArrow, ToArrow, UdfArrowConvert};
 use risingwave_common::array::Op;
-use risingwave_common::buffer::Bitmap;
+use risingwave_common::bitmap::Bitmap;
+use risingwave_pb::expr::PbUserDefinedFunctionMetadata;
 
 use super::*;
 use crate::sig::{UdfImpl, UdfKind, UdfOptions};
@@ -118,12 +119,10 @@ impl EstimateSize for State {
 impl AggStateDyn for State {}
 
 /// Create a new user-defined aggregate function.
-pub fn new_user_defined(agg: &AggCall) -> Result<BoxedAggregateFunction> {
-    let udf = agg
-        .user_defined
-        .as_ref()
-        .context("missing UDF definition")?;
-
+pub fn new_user_defined(
+    return_type: &DataType,
+    udf: &PbUserDefinedFunctionMetadata,
+) -> Result<BoxedAggregateFunction> {
     let identifier = udf.get_identifier()?;
     let language = udf.language.as_str();
     let runtime = udf.runtime.as_deref();
@@ -137,7 +136,7 @@ pub fn new_user_defined(agg: &AggCall) -> Result<BoxedAggregateFunction> {
         link: udf.link.as_deref(),
         identifier,
         arg_names: &udf.arg_names,
-        return_type: &agg.return_type,
+        return_type,
         always_retry_on_network_error: false,
         function_type: udf.function_type.as_deref(),
     })
@@ -154,9 +153,9 @@ pub fn new_user_defined(agg: &AggCall) -> Result<BoxedAggregateFunction> {
     ));
 
     Ok(Box::new(UserDefinedAggregateFunction {
-        return_field: arrow_convert.to_arrow_field("", &agg.return_type)?,
+        return_field: arrow_convert.to_arrow_field("", return_type)?,
         state_field: Field::new("state", arrow_schema::DataType::Binary, true),
-        return_type: agg.return_type.clone(),
+        return_type: return_type.clone(),
         arg_schema,
         runtime,
     }))
diff --git a/src/expr/core/src/lib.rs b/src/expr/core/src/lib.rs
index b250b8ce901f5..d45d4ca11f80a 100644
--- a/src/expr/core/src/lib.rs
+++ b/src/expr/core/src/lib.rs
@@ -15,7 +15,6 @@
 #![feature(let_chains)]
 #![feature(lint_reasons)]
 #![feature(iterator_try_collect)]
-#![feature(lazy_cell)]
 #![feature(coroutines)]
 #![feature(never_type)]
 #![feature(error_generic_member_access)]
diff --git a/src/expr/core/src/scalar/like.rs b/src/expr/core/src/scalar/like.rs
index 4f49f56cf43d9..d2261f5037fad 100644
--- a/src/expr/core/src/scalar/like.rs
+++ b/src/expr/core/src/scalar/like.rs
@@ -32,6 +32,12 @@ fn like_impl<const CASE_INSENSITIVE: bool>(
             let c = pbytes[px];
             match c {
                 b'_' => {
+                    if escape == b'_' {
+                        if px > 0 && pbytes[px - 1] != escape {
+                            px += 1;
+                            continue;
+                        }
+                    }
                     if sx < sbytes.len() {
                         px += 1;
                         sx += 1;
@@ -107,24 +113,15 @@ impl EscapeChar {
                 name: "escape",
                 reason: "only single ascii character is supported now".into(),
             })
-            .and_then(|c| {
-                // TODO: This is a temporary restriction since we don't output same result as PostgreSQL.
-                if c == b'_' {
-                    Err(ExprError::InvalidParam {
-                        name: "escape",
-                        reason: "`_` is not allowed as escape character now".into(),
-                    })
-                } else {
-                    Ok(c)
-                }
-            })
             .map(Self)
     }
 }
 
 #[cfg(test)]
 mod tests {
-    use super::{i_like_default, like_default};
+    use risingwave_expr::scalar::like::EscapeChar;
+
+    use super::{i_like_default, like, like_default};
 
     static CASES: &[(&str, &str, bool, bool)] = &[
         (r#"ABCDE"#, r#"%abcde%"#, false, false),
@@ -175,4 +172,26 @@ mod tests {
             );
         }
     }
+
+    static ESCAPE_CASES: &[(&str, &str, &str, bool)] = &[
+        (r"bear", r"b_ear", r"_", true),
+        (r"be_r", r"b_e__r", r"_", true),
+        (r"be__r", r"b_e___r", r"_", true),
+        (r"be___r", r"b_e____r", r"_", true),
+        (r"be_r", r"__e__r", r"_", false),
+        // TODO: Wrong behavior
+        (r"___r", r"____r", r"_", false),
+    ];
+
+    #[test]
+    fn test_escape_like() {
+        for (target, pattern, escape, expected) in ESCAPE_CASES {
+            let output = like(target, pattern, &EscapeChar::from_str(escape).unwrap());
+            assert_eq!(
+                output, *expected,
+                "target={}, pattern={}, escape={}",
+                target, pattern, escape
+            );
+        }
+    }
 }
diff --git a/src/expr/core/src/sig/mod.rs b/src/expr/core/src/sig/mod.rs
index 124a002f6519e..c3f57acd69f56 100644
--- a/src/expr/core/src/sig/mod.rs
+++ b/src/expr/core/src/sig/mod.rs
@@ -21,10 +21,11 @@ use std::sync::LazyLock;
 
 use itertools::Itertools;
 use risingwave_common::types::DataType;
+use risingwave_pb::expr::agg_call::PbType as AggregateFunctionType;
 use risingwave_pb::expr::expr_node::PbType as ScalarFunctionType;
 use risingwave_pb::expr::table_function::PbType as TableFunctionType;
 
-use crate::aggregate::{AggCall, AggKind as AggregateFunctionType, BoxedAggregateFunction};
+use crate::aggregate::{AggCall, BoxedAggregateFunction};
 use crate::error::Result;
 use crate::expr::BoxedExpression;
 use crate::table_function::BoxedTableFunction;
@@ -358,7 +359,7 @@ impl FuncName {
         match self {
             Self::Scalar(ty) => ty.as_str_name().into(),
             Self::Table(ty) => ty.as_str_name().into(),
-            Self::Aggregate(ty) => ty.to_protobuf().as_str_name().into(),
+            Self::Aggregate(ty) => ty.as_str_name().into(),
             Self::Udf(name) => name.clone().into(),
         }
     }
diff --git a/src/expr/core/src/window_function/kind.rs b/src/expr/core/src/window_function/kind.rs
index 2d0f261d99154..04b320f8ce9f2 100644
--- a/src/expr/core/src/window_function/kind.rs
+++ b/src/expr/core/src/window_function/kind.rs
@@ -19,7 +19,7 @@ use crate::aggregate::AggKind;
 use crate::Result;
 
 /// Kind of window functions.
-#[derive(Debug, Display, FromStr, Copy, Clone, PartialEq, Eq, Hash)]
+#[derive(Debug, Display, FromStr, Clone, PartialEq, Eq, Hash)]
 #[display(style = "snake_case")]
 pub enum WindowFuncKind {
     // General-purpose window functions.
@@ -52,7 +52,8 @@ impl WindowFuncKind {
                 Err(_) => bail!("no such window function type"),
             },
             PbType::Aggregate(agg_type) => match PbAggType::try_from(*agg_type) {
-                Ok(agg_type) => Self::Aggregate(AggKind::from_protobuf(agg_type)?),
+                // TODO(runji): support UDAF and wrapped scalar functions
+                Ok(agg_type) => Self::Aggregate(AggKind::from_protobuf(agg_type, None, None)?),
                 Err(_) => bail!("no such aggregate function type"),
             },
         };
diff --git a/src/expr/impl/Cargo.toml b/src/expr/impl/Cargo.toml
index 0f69c91e34162..a9c955f91dcca 100644
--- a/src/expr/impl/Cargo.toml
+++ b/src/expr/impl/Cargo.toml
@@ -18,7 +18,6 @@ normal = ["workspace-hack", "ctor"]
 [features]
 external-udf = ["arrow-udf-flight", "arrow-flight", "tonic"]
 js-udf = ["arrow-udf-js"]
-deno-udf = ["arrow-udf-js-deno", "zstd"]
 python-udf = ["arrow-udf-python"]
 wasm-udf = ["arrow-udf-wasm", "zstd"]
 
@@ -31,11 +30,11 @@ arrow-schema = { workspace = true }
 arrow-schema-iceberg = { workspace = true }
 arrow-udf-flight = { workspace = true, optional = true }
 arrow-udf-js = { workspace = true, optional = true }
-arrow-udf-js-deno = { workspace = true, optional = true }
 arrow-udf-python = { workspace = true, optional = true }
 arrow-udf-wasm = { workspace = true, optional = true }
 async-trait = "0.1"
 auto_enums = { workspace = true }
+bytes = "1"
 chrono = { version = "0.4", default-features = false, features = [
     "clock",
     "std",
diff --git a/src/expr/impl/benches/expr.rs b/src/expr/impl/benches/expr.rs
index 21036d3649acb..06cf18e4fbd0c 100644
--- a/src/expr/impl/benches/expr.rs
+++ b/src/expr/impl/benches/expr.rs
@@ -26,7 +26,7 @@ use criterion::{criterion_group, criterion_main, Criterion};
 use risingwave_common::array::*;
 use risingwave_common::types::test_utils::IntervalTestExt;
 use risingwave_common::types::*;
-use risingwave_expr::aggregate::{build_append_only, AggCall, AggKind};
+use risingwave_expr::aggregate::{build_append_only, AggCall, PbAggKind};
 use risingwave_expr::expr::*;
 use risingwave_expr::sig::FUNCTION_REGISTRY;
 use risingwave_pb::expr::expr_node::PbType;
@@ -386,7 +386,7 @@ fn bench_expr(c: &mut Criterion) {
     for sig in sigs {
         if matches!(
             sig.name.as_aggregate(),
-            AggKind::PercentileDisc | AggKind::PercentileCont
+            PbAggKind::PercentileDisc | PbAggKind::PercentileCont
         ) || (sig.inputs_type.iter())
             .chain([&sig.ret_type])
             .any(|t| !t.is_exact())
@@ -395,7 +395,7 @@ fn bench_expr(c: &mut Criterion) {
             continue;
         }
         let agg = match build_append_only(&AggCall {
-            kind: sig.name.as_aggregate(),
+            kind: sig.name.as_aggregate().into(),
             args: sig
                 .inputs_type
                 .iter()
@@ -406,7 +406,6 @@ fn bench_expr(c: &mut Criterion) {
             filter: None,
             distinct: false,
             direct_args: vec![],
-            user_defined: None,
         }) {
             Ok(agg) => agg,
             Err(e) => {
diff --git a/src/expr/impl/src/aggregate/approx_percentile.rs b/src/expr/impl/src/aggregate/approx_percentile.rs
new file mode 100644
index 0000000000000..1c5df73bc5c0f
--- /dev/null
+++ b/src/expr/impl/src/aggregate/approx_percentile.rs
@@ -0,0 +1,221 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::BTreeMap;
+use std::mem::size_of;
+use std::ops::Range;
+
+use bytes::{Buf, Bytes};
+use risingwave_common::array::*;
+use risingwave_common::row::Row;
+use risingwave_common::types::*;
+use risingwave_common_estimate_size::EstimateSize;
+use risingwave_expr::aggregate::{AggCall, AggStateDyn, AggregateFunction, AggregateState};
+use risingwave_expr::{build_aggregate, Result};
+
+/// TODO(kwannoel): for single phase agg, we can actually support `UDDSketch`.
+/// For two phase agg, we still use `DDSketch`.
+/// Then we also need to store the `relative_error` of the sketch, so we can report it
+/// in an internal table, if it changes.
+#[build_aggregate("approx_percentile(float8) -> float8", state = "bytea")]
+fn build(agg: &AggCall) -> Result<Box<dyn AggregateFunction>> {
+    let quantile = agg.direct_args[0]
+        .literal()
+        .map(|x| (*x.as_float64()).into())
+        .unwrap();
+    let relative_error: f64 = agg.direct_args[1]
+        .literal()
+        .map(|x| (*x.as_float64()).into())
+        .unwrap();
+    let base = (1.0 + relative_error) / (1.0 - relative_error);
+    Ok(Box::new(ApproxPercentile { quantile, base }))
+}
+
+#[allow(dead_code)]
+pub struct ApproxPercentile {
+    quantile: f64,
+    base: f64,
+}
+
+type BucketCount = u64;
+type BucketId = i32;
+type Count = u64;
+
+#[derive(Debug, Default)]
+struct State {
+    count: BucketCount,
+    pos_buckets: BTreeMap<BucketId, Count>,
+    zeros: Count,
+    neg_buckets: BTreeMap<BucketId, Count>,
+}
+
+impl EstimateSize for State {
+    fn estimated_heap_size(&self) -> usize {
+        let count_size = size_of::<BucketCount>();
+        let pos_buckets_size =
+            self.pos_buckets.len() * (size_of::<BucketId>() + size_of::<Count>());
+        let zero_bucket_size = size_of::<Count>();
+        let neg_buckets_size =
+            self.neg_buckets.len() * (size_of::<BucketId>() + size_of::<Count>());
+        count_size + pos_buckets_size + zero_bucket_size + neg_buckets_size
+    }
+}
+
+impl AggStateDyn for State {}
+
+impl ApproxPercentile {
+    fn add_datum(&self, state: &mut State, op: Op, datum: DatumRef<'_>) {
+        if let Some(value) = datum {
+            let prim_value = value.into_float64().into_inner();
+            let (non_neg, abs_value) = if prim_value < 0.0 {
+                (false, -prim_value)
+            } else {
+                (true, prim_value)
+            };
+            let bucket_id = abs_value.log(self.base).ceil() as BucketId;
+            match op {
+                Op::Delete | Op::UpdateDelete => {
+                    if abs_value == 0.0 {
+                        state.zeros -= 1;
+                    } else if non_neg {
+                        let count = state.pos_buckets.entry(bucket_id).or_insert(0);
+                        *count -= 1;
+                    } else {
+                        let count = state.neg_buckets.entry(bucket_id).or_insert(0);
+                        *count -= 1;
+                    }
+                    state.count -= 1;
+                }
+                Op::Insert | Op::UpdateInsert => {
+                    if abs_value == 0.0 {
+                        state.zeros += 1;
+                    } else if non_neg {
+                        let count = state.pos_buckets.entry(bucket_id).or_insert(0);
+                        *count += 1;
+                    } else {
+                        let count = state.neg_buckets.entry(bucket_id).or_insert(0);
+                        *count += 1;
+                    }
+                    state.count += 1;
+                }
+            }
+        };
+    }
+}
+
+#[async_trait::async_trait]
+impl AggregateFunction for ApproxPercentile {
+    fn return_type(&self) -> DataType {
+        DataType::Float64
+    }
+
+    fn create_state(&self) -> Result<AggregateState> {
+        Ok(AggregateState::Any(Box::<State>::default()))
+    }
+
+    async fn update(&self, state: &mut AggregateState, input: &StreamChunk) -> Result<()> {
+        let state: &mut State = state.downcast_mut();
+        for (op, row) in input.rows() {
+            let datum = row.datum_at(0);
+            self.add_datum(state, op, datum);
+        }
+        Ok(())
+    }
+
+    async fn update_range(
+        &self,
+        state: &mut AggregateState,
+        input: &StreamChunk,
+        range: Range<usize>,
+    ) -> Result<()> {
+        let state = state.downcast_mut();
+        for (op, row) in input.rows_in(range) {
+            self.add_datum(state, op, row.datum_at(0));
+        }
+        Ok(())
+    }
+
+    // TODO(kwannoel): Instead of iterating over all buckets, we can maintain the
+    // approximate quantile bucket on the fly.
+    async fn get_result(&self, state: &AggregateState) -> Result<Datum> {
+        let state = state.downcast_ref::<State>();
+        let quantile_count = (state.count as f64 * self.quantile).floor() as u64;
+        let mut acc_count = 0;
+        for (bucket_id, count) in state.neg_buckets.iter().rev() {
+            acc_count += count;
+            if acc_count > quantile_count {
+                // approx value = -2 * y^i / (y + 1)
+                let approx_percentile = -2.0 * self.base.powi(*bucket_id) / (self.base + 1.0);
+                let approx_percentile = ScalarImpl::Float64(approx_percentile.into());
+                return Ok(Datum::from(approx_percentile));
+            }
+        }
+        acc_count += state.zeros;
+        if acc_count > quantile_count {
+            return Ok(Datum::from(ScalarImpl::Float64(0.0.into())));
+        }
+        for (bucket_id, count) in &state.pos_buckets {
+            acc_count += count;
+            if acc_count > quantile_count {
+                // approx value = 2 * y^i / (y + 1)
+                let approx_percentile = 2.0 * self.base.powi(*bucket_id) / (self.base + 1.0);
+                let approx_percentile = ScalarImpl::Float64(approx_percentile.into());
+                return Ok(Datum::from(approx_percentile));
+            }
+        }
+        return Ok(None);
+    }
+
+    fn encode_state(&self, state: &AggregateState) -> Result<Datum> {
+        let state = state.downcast_ref::<State>();
+        let mut encoded_state = Vec::with_capacity(state.estimated_heap_size());
+        encoded_state.extend_from_slice(&state.count.to_be_bytes());
+        encoded_state.extend_from_slice(&state.zeros.to_be_bytes());
+        let neg_buckets_size = state.neg_buckets.len() as u64;
+        encoded_state.extend_from_slice(&neg_buckets_size.to_be_bytes());
+        for (bucket_id, count) in &state.neg_buckets {
+            encoded_state.extend_from_slice(&bucket_id.to_be_bytes());
+            encoded_state.extend_from_slice(&count.to_be_bytes());
+        }
+        for (bucket_id, count) in &state.pos_buckets {
+            encoded_state.extend_from_slice(&bucket_id.to_be_bytes());
+            encoded_state.extend_from_slice(&count.to_be_bytes());
+        }
+        let encoded_scalar = ScalarImpl::Bytea(encoded_state.into());
+        Ok(Datum::from(encoded_scalar))
+    }
+
+    fn decode_state(&self, datum: Datum) -> Result<AggregateState> {
+        let mut state = State::default();
+        let Some(scalar_state) = datum else {
+            return Ok(AggregateState::Any(Box::new(state)));
+        };
+        let encoded_state: Box<[u8]> = scalar_state.into_bytea();
+        let mut buf = Bytes::from(encoded_state);
+        state.count = buf.get_u64();
+        state.zeros = buf.get_u64();
+        let neg_buckets_size = buf.get_u64();
+        for _ in 0..neg_buckets_size {
+            let bucket_id = buf.get_i32();
+            let count = buf.get_u64();
+            state.neg_buckets.insert(bucket_id, count);
+        }
+        while !buf.is_empty() {
+            let bucket_id = buf.get_i32();
+            let count = buf.get_u64();
+            state.pos_buckets.insert(bucket_id, count);
+        }
+        Ok(AggregateState::Any(Box::new(state)))
+    }
+}
diff --git a/src/expr/impl/src/aggregate/mod.rs b/src/expr/impl/src/aggregate/mod.rs
index c0b6a5ae64c31..349574018fedf 100644
--- a/src/expr/impl/src/aggregate/mod.rs
+++ b/src/expr/impl/src/aggregate/mod.rs
@@ -13,6 +13,7 @@
 // limitations under the License.
 
 mod approx_count_distinct;
+mod approx_percentile;
 mod array_agg;
 mod bit_and;
 mod bit_or;
diff --git a/src/expr/impl/src/lib.rs b/src/expr/impl/src/lib.rs
index 56bdbe3b81100..e5c69c2660eeb 100644
--- a/src/expr/impl/src/lib.rs
+++ b/src/expr/impl/src/lib.rs
@@ -25,8 +25,6 @@
 #![feature(assert_matches)]
 #![feature(lint_reasons)]
 #![feature(iterator_try_collect)]
-#![feature(exclusive_range_pattern)]
-#![feature(lazy_cell)]
 #![feature(coroutines)]
 #![feature(test)]
 #![feature(iter_array_chunks)]
diff --git a/src/expr/impl/src/scalar/cast.rs b/src/expr/impl/src/scalar/cast.rs
index 48218d7200809..0c93c0ed15dd9 100644
--- a/src/expr/impl/src/scalar/cast.rs
+++ b/src/expr/impl/src/scalar/cast.rs
@@ -147,8 +147,8 @@ pub fn int_to_bool(input: i32) -> bool {
     input != 0
 }
 
-// For most of the types, cast them to varchar is similar to return their text format.
-// So we use this function to cast type to varchar.
+/// For most of the types, cast them to varchar is the same as their pgwire "TEXT" format.
+/// So we use `ToText` to cast type to varchar.
 #[function("cast(*int) -> varchar")]
 #[function("cast(decimal) -> varchar")]
 #[function("cast(*float) -> varchar")]
@@ -177,7 +177,7 @@ pub fn bool_to_varchar(input: bool, writer: &mut impl Write) {
         .unwrap();
 }
 
-/// `bool_out` is different from `general_to_string<bool>` to produce a single char. `PostgreSQL`
+/// `bool_out` is different from `cast(boolean) -> varchar` to produce a single char. `PostgreSQL`
 /// uses different variants of bool-to-string in different situations.
 #[function("bool_out(boolean) -> varchar")]
 pub fn bool_out(input: bool, writer: &mut impl Write) {
diff --git a/src/expr/impl/src/scalar/cmp.rs b/src/expr/impl/src/scalar/cmp.rs
index 02c1662f18025..20c0afc307e74 100644
--- a/src/expr/impl/src/scalar/cmp.rs
+++ b/src/expr/impl/src/scalar/cmp.rs
@@ -15,7 +15,7 @@
 use std::fmt::Debug;
 
 use risingwave_common::array::{Array, BoolArray};
-use risingwave_common::buffer::Bitmap;
+use risingwave_common::bitmap::Bitmap;
 use risingwave_common::row::Row;
 use risingwave_common::types::{Scalar, ScalarRef, ScalarRefImpl};
 use risingwave_expr::function;
diff --git a/src/expr/impl/src/scalar/trigonometric.rs b/src/expr/impl/src/scalar/trigonometric.rs
index 20e2b3523716b..325fc85538ffe 100644
--- a/src/expr/impl/src/scalar/trigonometric.rs
+++ b/src/expr/impl/src/scalar/trigonometric.rs
@@ -95,22 +95,47 @@ pub fn atanh_f64(input: F64) -> F64 {
     f64::atanh(input.0).into()
 }
 
-// Radians per degree, a.k.a. PI / 180
+static DEGREE_THIRTY: f64 = 30.0;
+static DEGREE_FORTY_FIVE: f64 = 45.0;
+static DEGREE_SIXTY: f64 = 60.0;
+static DEGREE_ONE_HALF: f64 = 0.5;
+static DEGREE_ONE: f64 = 1.0;
 static RADIANS_PER_DEGREE: f64 = 0.017_453_292_519_943_295;
+
 // Constants we use to get more accurate results.
+// Depend on the machine and have to be evaluated at runtime
 // See PSQL: https://github.com/postgres/postgres/blob/78ec02d612a9b69039ec2610740f738968fe144d/src/backend/utils/adt/float.c#L2024
-static SIND_30: f64 = 0.499_999_999_999_999_94;
-static ONE_MINUS_COSD_60: f64 = 0.499_999_999_999_999_9;
-static TAND_45: f64 = 1.0;
-static COTD_45: f64 = 1.0;
-static ASIN_0_5: f64 = 0.523_598_775_598_298_8;
+fn sind_30() -> f64 {
+    f64::sin(DEGREE_THIRTY * RADIANS_PER_DEGREE)
+}
+
+fn one_minus_cosd_60() -> f64 {
+    DEGREE_ONE - f64::cos(DEGREE_SIXTY * RADIANS_PER_DEGREE)
+}
+
+fn tand_45() -> f64 {
+    f64::tan(DEGREE_FORTY_FIVE * RADIANS_PER_DEGREE)
+}
+
+fn cotd_45() -> f64 {
+    f64::cos(DEGREE_FORTY_FIVE * RADIANS_PER_DEGREE)
+        / f64::sin(DEGREE_FORTY_FIVE * RADIANS_PER_DEGREE)
+}
+
+fn asin_0_5() -> f64 {
+    f64::asin(DEGREE_ONE_HALF)
+}
+
+fn acos_0_5() -> f64 {
+    f64::acos(DEGREE_ONE_HALF)
+}
 
 // returns the cosine of an angle that lies between 0 and 60 degrees. This will return exactly 1
 // when xi s 0, and exactly 0.5 when x is 60 degrees.
 fn cosd_0_to_60(x: f64) -> f64 {
     // https://github.com/postgres/postgres/blob/REL_15_2/src/backend/utils/adt/float.c
     let one_minus_cos_x: f64 = 1.0 - f64::cos(x * RADIANS_PER_DEGREE);
-    1.0 - (one_minus_cos_x / ONE_MINUS_COSD_60) / 2.0
+    1.0 - (one_minus_cos_x / one_minus_cosd_60()) / 2.0
 }
 
 // returns the sine of an angle that lies between 0 and 30 degrees. This will return exactly 0 when
@@ -118,7 +143,7 @@ fn cosd_0_to_60(x: f64) -> f64 {
 fn sind_0_to_30(x: f64) -> f64 {
     // https://github.com/postgres/postgres/blob/REL_15_2/src/backend/utils/adt/float.c
     let sin_x = f64::sin(x * RADIANS_PER_DEGREE);
-    (sin_x / SIND_30) / 2.0
+    (sin_x / sind_30()) / 2.0
 }
 
 // returns the cosine of an angle in the first quadrant (0 to 90 degrees).
@@ -237,6 +262,20 @@ pub fn cotd_f64(input: F64) -> F64 {
     let mut arg1 = input.0 % 360.0;
     let mut sign = 1.0;
 
+    // hardcoding exact results.
+    if arg1 == 45.0 {
+        return F64::from(1.0);
+    }
+    if arg1 == 135.0 {
+        return F64::from(-1.0);
+    }
+    if arg1 == 225. {
+        return F64::from(1.0);
+    }
+    if arg1 == 315.0 {
+        return F64::from(-1.0);
+    }
+
     if arg1 < 0.0 {
         // cotd(-x) = -cotd(x)
         arg1 = -arg1;
@@ -256,7 +295,7 @@ pub fn cotd_f64(input: F64) -> F64 {
     }
 
     let cot_arg1 = cosd_q1(arg1) / sind_q1(arg1);
-    let result = sign * (cot_arg1 / COTD_45);
+    let result = sign * (cot_arg1 / cotd_45());
 
     // On some machines we get cotd(270) = minus zero, but this isn't always
     // true. For portability, and because the user constituency for this
@@ -277,6 +316,20 @@ pub fn tand_f64(input: F64) -> F64 {
     let mut arg1 = input.0 % 360.0;
     let mut sign = 1.0;
 
+    // hardcoding exact results.
+    if arg1 == 45.0 {
+        return F64::from(1.0);
+    }
+    if arg1 == 135.0 {
+        return F64::from(-1.0);
+    }
+    if arg1 == 225. {
+        return F64::from(1.0);
+    }
+    if arg1 == 315.0 {
+        return F64::from(-1.0);
+    }
+
     if arg1 < 0.0 {
         // tand(-x) = -tand(x)
         arg1 = -arg1;
@@ -296,7 +349,7 @@ pub fn tand_f64(input: F64) -> F64 {
     }
 
     let tan_arg1 = sind_q1(arg1) / cosd_q1(arg1);
-    let result = sign * (tan_arg1 / TAND_45);
+    let result = sign * (tan_arg1 / tand_45());
 
     // On some machines we get tand(180) = minus zero, but this isn't always true. For portability,
     // and because the user constituency for this function probably doesn't want minus zero, force
@@ -315,11 +368,11 @@ pub fn asind_q1(x: f64) -> f64 {
     // monotonic functionover the full range.
     if x <= 0.5 {
         let asin_x = f64::asin(x);
-        return (asin_x / ASIN_0_5) * 30.0;
+        return (asin_x / asin_0_5()) * 30.0;
     }
 
     let acos_x = f64::acos(x);
-    90.0 - (acos_x / ASIN_0_5) * 60.0
+    90.0 - (acos_x / acos_0_5()) * 60.0
 }
 
 #[function("asind(float8) -> float8")]
@@ -332,7 +385,7 @@ pub fn asind_f64(input: F64) -> F64 {
     }
 
     // Return NaN if input is out of range. Slightly different from PSQL implementation
-    if input.0 < -1.0 || input.0 > 1.0 {
+    if !(-1.0..=1.0).contains(&arg1) {
         return F64::from(f64::NAN);
     }
 
@@ -348,6 +401,42 @@ pub fn asind_f64(input: F64) -> F64 {
     result.into()
 }
 
+// returns the inverse cosine of x in degrees, for x in the range [0, 1].  The result is an angle in
+// the first quadrant --- [0, 90] degrees. For the 3 special case inputs (0, 0.5 and 1), this
+// function will return exact values (0, 60 and 90 degrees respectively).
+fn acosd_q1(x: f64) -> f64 {
+    // Stitch together inverse sine and cosine functions for the ranges [0, 0.5] and (0.5, 1].  Each
+    // expression below is guaranteed to return exactly 60 for x=0.5, so the result is a continuous
+    // monotonic function over the full range.
+    if x <= 0.5 {
+        let asin_x = f64::asin(x);
+        return 90.0 - (asin_x / asin_0_5()) * 30.0;
+    }
+    let acos_x = f64::acos(x);
+    (acos_x / acos_0_5()) * 60.0
+}
+
+#[function("acosd(float8) -> float8")]
+pub fn acosd_f64(input: F64) -> F64 {
+    let arg1 = input.0;
+
+    // Return NaN if input is NaN or Infinite. Slightly different from PSQL implementation
+    if input.0.is_nan() || input.0.is_infinite() || !(-1.0..=1.0).contains(&arg1) {
+        return F64::from(f64::NAN);
+    }
+
+    let result = if arg1 >= 0.0 {
+        acosd_q1(arg1)
+    } else {
+        90.0 + asind_q1(-arg1)
+    };
+
+    if result.is_infinite() {
+        return F64::from(f64::NAN);
+    }
+    result.into()
+}
+
 #[function("degrees(float8) -> float8")]
 pub fn degrees_f64(input: F64) -> F64 {
     input.0.to_degrees().into()
@@ -360,18 +449,19 @@ pub fn radians_f64(input: F64) -> F64 {
 
 #[cfg(test)]
 mod tests {
-    use std::f64::consts::PI;
-
     use risingwave_common::types::FloatExt;
 
     use crate::scalar::trigonometric::*;
 
     fn precision() -> f64 {
-        1e-13
+        1e-12
     }
 
     /// numbers are equal within a rounding error
     fn assert_similar(lhs: F64, rhs: F64) {
+        if lhs == F64::from(f64::NAN) && rhs == F64::from(f64::NAN) {
+            return;
+        }
         let x = (lhs.0 - rhs.0).abs() <= precision();
         assert!(
             x,
@@ -385,7 +475,7 @@ mod tests {
     #[test]
     fn test_degrees() {
         let d = F64::from(180);
-        let pi = F64::from(PI);
+        let pi = F64::from(core::f64::consts::PI);
 
         // sind
         assert_similar(sin_f64(50_f64.to_radians().into()), sind_f64(F64::from(50)));
@@ -454,7 +544,6 @@ mod tests {
             tand_f64(F64::from(-10)),
         );
         assert_similar(tan_f64(50_f64.to_radians().into()), tand_f64(F64::from(50)));
-        // we get slightly different result here, which is why I reduce the required accuracy
         assert!(
             (tan_f64(250_f64.to_radians().into()) - tand_f64(F64::from(250)))
                 .0
@@ -471,8 +560,17 @@ mod tests {
         assert_similar(asind_f64(F64::from(-0.5)), F64::from(-30));
         assert_similar(asind_f64(F64::from(0)), F64::from(0));
         assert_similar(asind_f64(F64::from(0.5)), F64::from(30));
+        assert_similar(asind_f64(F64::from(0.75)), F64::from(48.590377890729));
         assert_similar(asind_f64(F64::from(1)), F64::from(90));
 
+        // acosd
+        assert_eq!(acosd_f64(F64::from(-1)), F64::from(180));
+        assert_similar(acosd_f64(F64::from(-0.75)), F64::from(138.59037789072914));
+        assert_eq!(acosd_f64(F64::from(-0.5)), F64::from(120));
+        assert_eq!(acosd_f64(F64::from(0.0)), F64::from(90));
+        assert_eq!(acosd_f64(F64::from(0.5)), F64::from(60));
+        assert_eq!(acosd_f64(F64::from(1)), F64::from(0));
+
         // exact matches
         assert!(tand_f64(F64::from(-270)).0.is_infinite());
         assert_eq!(tand_f64(F64::from(-180)), 0.0);
@@ -576,4 +674,16 @@ mod tests {
             atanh_f64(F64::from(x.powi(2) - 1.0) / F64::from(x.powi(2) + 1.0)),
         );
     }
+
+    #[test]
+    fn test_exact() {
+        assert_eq!(cotd_f64(F64::from(135.0)).0, -1.0);
+        assert_eq!(cotd_f64(F64::from(225.0)).0, 1.0);
+        assert_eq!(cotd_f64(F64::from(315.0)).0, -1.0);
+        assert_eq!(cotd_f64(F64::from(45.0)).0, 1.0);
+        assert_eq!(tand_f64(F64::from(45.0)).0, 1.0);
+        assert_eq!(tand_f64(F64::from(135.0)).0, -1.0);
+        assert_eq!(tand_f64(F64::from(225.0)).0, 1.0);
+        assert_eq!(tand_f64(F64::from(315.0)).0, -1.0);
+    }
 }
diff --git a/src/expr/impl/src/udf/deno.rs b/src/expr/impl/src/udf/deno.rs
deleted file mode 100644
index 8c0c0a4674a8c..0000000000000
--- a/src/expr/impl/src/udf/deno.rs
+++ /dev/null
@@ -1,127 +0,0 @@
-// Copyright 2024 RisingWave Labs
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::sync::Arc;
-
-use anyhow::bail;
-use arrow_udf_js_deno::{CallMode, Runtime};
-use futures_async_stream::try_stream;
-use risingwave_common::array::arrow::{ToArrow, UdfArrowConvert};
-
-use super::*;
-
-#[linkme::distributed_slice(UDF_IMPLS)]
-static DENO: UdfImplDescriptor = UdfImplDescriptor {
-    match_fn: |language, runtime, _link| language == "javascript" && runtime == Some("deno"),
-    create_fn: |opts| {
-        let mut body = None;
-        let mut compressed_binary = None;
-        let identifier = opts.name.to_string();
-        match (opts.using_link, opts.using_base64_decoded, opts.as_) {
-            (None, None, Some(as_)) => body = Some(as_.to_string()),
-            (Some(link), None, None) => {
-                let bytes = read_file_from_link(link)?;
-                compressed_binary = Some(zstd::stream::encode_all(bytes.as_slice(), 0)?);
-            }
-            (None, Some(bytes), None) => {
-                compressed_binary = Some(zstd::stream::encode_all(bytes, 0)?);
-            }
-            (None, None, None) => bail!("Either USING or AS must be specified"),
-            _ => bail!("Both USING and AS cannot be specified"),
-        }
-        Ok(CreateFunctionOutput {
-            identifier,
-            body,
-            compressed_binary,
-        })
-    },
-    build_fn: |opts| {
-        let runtime = Runtime::new();
-        let body = match (opts.body, opts.compressed_binary) {
-            (Some(body), _) => body.to_string(),
-            (_, Some(compressed_binary)) => {
-                let binary = zstd::stream::decode_all(compressed_binary)
-                    .context("failed to decompress binary")?;
-                String::from_utf8(binary).context("failed to decode binary")?
-            }
-            _ => bail!("UDF body or compressed binary is required for deno UDF"),
-        };
-
-        let body = format!(
-            "export {} {}({}) {{ {} }}",
-            match opts.function_type {
-                Some("sync") => "function",
-                Some("async") => "async function",
-                Some("generator") => "function*",
-                Some("async_generator") => "async function*",
-                _ if opts.kind.is_table() => "function*",
-                _ => "function",
-            },
-            opts.identifier,
-            opts.arg_names.join(","),
-            body
-        );
-
-        tokio::task::block_in_place(|| {
-            tokio::runtime::Handle::current().block_on(runtime.add_function(
-                opts.identifier,
-                UdfArrowConvert::default().to_arrow_field("", opts.return_type)?,
-                CallMode::CalledOnNullInput,
-                &body,
-            ))
-        })?;
-
-        Ok(Box::new(DenoFunction {
-            runtime,
-            identifier: opts.identifier.to_string(),
-        }))
-    },
-};
-
-#[derive(Debug)]
-struct DenoFunction {
-    runtime: Arc<Runtime>,
-    identifier: String,
-}
-
-#[async_trait::async_trait]
-impl UdfImpl for DenoFunction {
-    async fn call(&self, input: &RecordBatch) -> Result<RecordBatch> {
-        // FIXME(runji): make the future Send
-        tokio::task::block_in_place(|| {
-            tokio::runtime::Handle::current()
-                .block_on(self.runtime.call(&self.identifier, input.clone()))
-        })
-    }
-
-    async fn call_table_function<'a>(
-        &'a self,
-        input: &'a RecordBatch,
-    ) -> Result<BoxStream<'a, Result<RecordBatch>>> {
-        Ok(self.call_table_function_inner(input.clone()))
-    }
-}
-
-impl DenoFunction {
-    #[try_stream(boxed, ok = RecordBatch, error = anyhow::Error)]
-    async fn call_table_function_inner<'a>(&'a self, input: RecordBatch) {
-        let mut stream = self
-            .runtime
-            .call_table_function(&self.identifier, input, 1024)
-            .await?;
-        while let Some(batch) = stream.next().await {
-            yield batch?;
-        }
-    }
-}
diff --git a/src/expr/impl/src/udf/mod.rs b/src/expr/impl/src/udf/mod.rs
index 8ae9b6e98b69f..599fe2cb5198f 100644
--- a/src/expr/impl/src/udf/mod.rs
+++ b/src/expr/impl/src/udf/mod.rs
@@ -22,8 +22,6 @@ use risingwave_expr::sig::{
     CreateFunctionOptions, CreateFunctionOutput, UdfImpl, UdfImplDescriptor, UDF_IMPLS,
 };
 
-#[cfg(feature = "deno-udf")]
-mod deno;
 #[cfg(feature = "external-udf")]
 #[cfg(not(madsim))]
 mod external;
diff --git a/src/expr/impl/src/window_function/aggregate.rs b/src/expr/impl/src/window_function/aggregate.rs
index 9942581357f77..da30d35a4e3e8 100644
--- a/src/expr/impl/src/window_function/aggregate.rs
+++ b/src/expr/impl/src/window_function/aggregate.rs
@@ -21,7 +21,7 @@ use risingwave_common::util::iter_util::ZipEqFast;
 use risingwave_common::{bail, must_match};
 use risingwave_common_estimate_size::{EstimateSize, KvSize};
 use risingwave_expr::aggregate::{
-    AggCall, AggregateFunction, AggregateState as AggImplState, BoxedAggregateFunction,
+    AggCall, AggKind, AggregateFunction, AggregateState as AggImplState, BoxedAggregateFunction,
 };
 use risingwave_expr::sig::FUNCTION_REGISTRY;
 use risingwave_expr::window_function::{
@@ -50,10 +50,10 @@ pub(super) fn new(call: &WindowFuncCall) -> Result<BoxedWindowState> {
     if call.frame.bounds.validate().is_err() {
         bail!("the window frame must be valid");
     }
-    let agg_kind = must_match!(call.kind, WindowFuncKind::Aggregate(agg_kind) => agg_kind);
+    let agg_kind = must_match!(&call.kind, WindowFuncKind::Aggregate(agg_kind) => agg_kind);
     let arg_data_types = call.args.arg_types().to_vec();
     let agg_call = AggCall {
-        kind: agg_kind,
+        kind: agg_kind.clone(),
         args: call.args.clone(),
         return_type: call.return_type.clone(),
         column_orders: Vec::new(), // the input is already sorted
@@ -62,10 +62,11 @@ pub(super) fn new(call: &WindowFuncCall) -> Result<BoxedWindowState> {
         // TODO(rc): support distinct on window function call? PG doesn't support it either.
         distinct: false,
         direct_args: vec![],
-        user_defined: None,
     };
+    // TODO(runji): support UDAF and wrapped scalar function
+    let agg_kind = must_match!(agg_kind, AggKind::Builtin(agg_kind) => agg_kind);
     let agg_func_sig = FUNCTION_REGISTRY
-        .get(agg_kind, &arg_data_types, &call.return_type)
+        .get(*agg_kind, &arg_data_types, &call.return_type)
         .expect("the agg func must exist");
     let agg_func = agg_func_sig.build_aggregate(&agg_call)?;
     let (agg_impl, enable_delta) =
diff --git a/src/expr/impl/src/window_function/buffer.rs b/src/expr/impl/src/window_function/buffer.rs
index bd1c10d162b23..57217dda6fd4b 100644
--- a/src/expr/impl/src/window_function/buffer.rs
+++ b/src/expr/impl/src/window_function/buffer.rs
@@ -970,7 +970,7 @@ mod tests {
 
         let key = |key: i64| -> StateKey {
             StateKey {
-                order_key: memcmp_encoding::encode_value(&Some(ScalarImpl::from(key)), order_type)
+                order_key: memcmp_encoding::encode_value(Some(ScalarImpl::from(key)), order_type)
                     .unwrap(),
                 pk: OwnedRow::empty().into(),
             }
diff --git a/src/expr/impl/src/window_function/mod.rs b/src/expr/impl/src/window_function/mod.rs
index 1153a9111a1ea..38c299a01b9d5 100644
--- a/src/expr/impl/src/window_function/mod.rs
+++ b/src/expr/impl/src/window_function/mod.rs
@@ -28,7 +28,7 @@ fn create_window_state_impl(call: &WindowFuncCall) -> Result<BoxedWindowState> {
     assert!(call.frame.bounds.validate().is_ok());
 
     use WindowFuncKind::*;
-    Ok(match call.kind {
+    Ok(match &call.kind {
         RowNumber => Box::new(rank::RankState::<rank::RowNumber>::new(call)),
         Rank => Box::new(rank::RankState::<rank::Rank>::new(call)),
         DenseRank => Box::new(rank::RankState::<rank::DenseRank>::new(call)),
diff --git a/src/expr/macro/src/context.rs b/src/expr/macro/src/context.rs
index afae444e03c45..47b0638284435 100644
--- a/src/expr/macro/src/context.rs
+++ b/src/expr/macro/src/context.rs
@@ -16,7 +16,7 @@ use itertools::Itertools;
 use proc_macro2::TokenStream;
 use quote::{quote, quote_spanned, ToTokens};
 use syn::parse::{Parse, ParseStream};
-use syn::{Error, FnArg, Ident, ItemFn, Result, Token, Type, Visibility};
+use syn::{Error, FnArg, Ident, ItemFn, Pat, PatType, Result, ReturnType, Token, Type, Visibility};
 
 use crate::utils::extend_vis_with_super;
 
@@ -138,7 +138,47 @@ pub(super) fn generate_captured_function(
     mut user_fn: ItemFn,
 ) -> Result<TokenStream> {
     let CaptureContextAttr { captures } = attr;
-    let orig_user_fn = user_fn.clone();
+    let is_async = user_fn.sig.asyncness.is_some();
+    let mut orig_user_fn = user_fn.clone();
+    if is_async {
+        // Modify the return type to impl Future<Output = output> + Send + 'static for the original function.
+        let output_type = match &orig_user_fn.sig.output {
+            ReturnType::Type(_, ty) => ty.clone(),
+            ReturnType::Default => Box::new(syn::parse_quote!(())),
+        };
+        orig_user_fn.sig.output = ReturnType::Type(
+            syn::token::RArrow::default(),
+            Box::new(
+                syn::parse_quote!(impl std::future::Future<Output = #output_type> + Send + 'static),
+            ),
+        );
+        orig_user_fn.sig.asyncness = None;
+
+        // Generate clone statements for each input
+        let input_def: Vec<TokenStream> = orig_user_fn
+            .sig
+            .inputs
+            .iter()
+            .map(|arg| {
+                if let FnArg::Typed(PatType { pat, .. }) = arg {
+                    if let Pat::Ident(ident) = pat.as_ref() {
+                        let ident_name = &ident.ident;
+                        return quote! {
+                            let #ident_name = #ident_name.clone();
+                        };
+                    }
+                }
+                quote! {}
+            })
+            .collect();
+
+        // Wrap the original function body in async move { ... }.
+        let orig_body = &orig_user_fn.block;
+        orig_user_fn.block = Box::new(syn::parse_quote!({
+            #(#input_def)*
+            async move { #orig_body }
+        }));
+    }
 
     let sig = &mut user_fn.sig;
 
@@ -151,6 +191,11 @@ pub(super) fn generate_captured_function(
         sig.ident = new_name;
     }
 
+    if is_async {
+        // Ensure the function is async
+        sig.asyncness = Some(syn::token::Async::default());
+    }
+
     // Modify the inputs of sig.
     let inputs = &mut sig.inputs;
     if inputs.len() < captures.len() {
@@ -200,21 +245,33 @@ pub(super) fn generate_captured_function(
                 ));
             };
             let name = arg.pat.into_token_stream();
-            scoped = quote_spanned! { context.span()=>
-                // TODO: Can we add an assertion here that `&<<#context::Type> as Deref>::Target` is same as `#arg.ty`?
-                #context::try_with(|#name| {
-                    #scoped
-                }).flatten()
-            }
+            // TODO: Can we add an assertion here that `&<<#context::Type> as Deref>::Target` is same as `#arg.ty`?
+            scoped = if is_async {
+                quote_spanned! { context.span()=>
+                    #context::try_with(|#name| { #scoped })
+                }
+            } else {
+                quote_spanned! { context.span()=>
+                    #context::try_with(|#name| { #scoped }).flatten()
+                }
+            };
         }
         scoped
     };
     let new_user_fn = {
         let vis = user_fn.vis;
         let sig = user_fn.sig;
-        quote! {
-            #vis #sig {
-                {#new_body}.map_err(Into::into)
+        if is_async {
+            quote! {
+                #vis #sig {
+                    {#new_body}?.await
+                }
+            }
+        } else {
+            quote! {
+                #vis #sig {
+                    {#new_body}.map_err(Into::into)
+                }
             }
         }
     };
diff --git a/src/expr/macro/src/gen.rs b/src/expr/macro/src/gen.rs
index 8bf39c17e763b..ba51f4ba6bf54 100644
--- a/src/expr/macro/src/gen.rs
+++ b/src/expr/macro/src/gen.rs
@@ -561,7 +561,7 @@ impl FunctionAttr {
                 use std::sync::Arc;
                 use risingwave_common::array::*;
                 use risingwave_common::types::*;
-                use risingwave_common::buffer::Bitmap;
+                use risingwave_common::bitmap::Bitmap;
                 use risingwave_common::row::OwnedRow;
                 use risingwave_common::util::iter_util::ZipEqFast;
 
@@ -695,7 +695,7 @@ impl FunctionAttr {
                 use risingwave_expr::sig::{FuncSign, SigDataType, FuncBuilder};
 
                 FuncSign {
-                    name: risingwave_expr::aggregate::AggKind::#pb_type.into(),
+                    name: risingwave_pb::expr::agg_call::Type::#pb_type.into(),
                     inputs_type: vec![#(#args),*],
                     variadic: false,
                     ret_type: #ret,
@@ -914,7 +914,7 @@ impl FunctionAttr {
                 use risingwave_common::array::*;
                 use risingwave_common::types::*;
                 use risingwave_common::bail;
-                use risingwave_common::buffer::Bitmap;
+                use risingwave_common::bitmap::Bitmap;
                 use risingwave_common_estimate_size::EstimateSize;
 
                 use risingwave_expr::expr::Context;
@@ -1195,7 +1195,7 @@ impl FunctionAttr {
             |return_type, chunk_size, children| {
                 use risingwave_common::array::*;
                 use risingwave_common::types::*;
-                use risingwave_common::buffer::Bitmap;
+                use risingwave_common::bitmap::Bitmap;
                 use risingwave_common::util::iter_util::ZipEqFast;
                 use risingwave_expr::expr::{BoxedExpression, Context};
                 use risingwave_expr::{Result, ExprError};
diff --git a/src/expr/macro/src/lib.rs b/src/expr/macro/src/lib.rs
index ceaef0d6d2406..3a905165c2ee2 100644
--- a/src/expr/macro/src/lib.rs
+++ b/src/expr/macro/src/lib.rs
@@ -637,6 +637,8 @@ pub fn capture_context(attr: TokenStream, item: TokenStream) -> TokenStream {
     fn inner(attr: TokenStream, item: TokenStream) -> Result<TokenStream2> {
         let attr: CaptureContextAttr = syn::parse(attr)?;
         let user_fn: ItemFn = syn::parse(item)?;
+
+        // Generate captured function
         generate_captured_function(attr, user_fn)
     }
     match inner(attr, item) {
diff --git a/src/frontend/Cargo.toml b/src/frontend/Cargo.toml
index a59ce2e55f678..89d29e076a38f 100644
--- a/src/frontend/Cargo.toml
+++ b/src/frontend/Cargo.toml
@@ -26,6 +26,7 @@ auto_impl = "1"
 base64 = "0.22"
 bk-tree = "0.5.0"
 bytes = "1"
+chrono = { version = "0.4", default-features = false }
 clap = { workspace = true }
 downcast-rs = "1.2"
 dyn-clone = "1.0.14"
@@ -38,6 +39,7 @@ fixedbitset = "0.5"
 futures = { version = "0.3", default-features = false, features = ["alloc"] }
 futures-async-stream = { workspace = true }
 iana-time-zone = "0.1"
+iceberg = { workspace = true }
 icelake = { workspace = true }
 itertools = { workspace = true }
 jsonbb = { workspace = true }
diff --git a/src/frontend/planner_test/src/lib.rs b/src/frontend/planner_test/src/lib.rs
index ee4f942f09889..abb291cc37d7e 100644
--- a/src/frontend/planner_test/src/lib.rs
+++ b/src/frontend/planner_test/src/lib.rs
@@ -36,7 +36,7 @@ use risingwave_frontend::session::SessionImpl;
 use risingwave_frontend::test_utils::{create_proto_file, get_explain_output, LocalFrontend};
 use risingwave_frontend::{
     build_graph, explain_stream_graph, Binder, Explain, FrontendOpts, OptimizerContext,
-    OptimizerContextRef, PlanRef, Planner, WithOptions,
+    OptimizerContextRef, PlanRef, Planner, WithOptionsSecResolved,
 };
 use risingwave_sqlparser::ast::{
     AstOption, DropMode, EmitMode, ExplainOptions, ObjectName, Statement,
@@ -836,7 +836,8 @@ impl TestCase {
                 let mut options = BTreeMap::new();
                 options.insert("connector".to_string(), "blackhole".to_string());
                 options.insert("type".to_string(), "append-only".to_string());
-                let options = WithOptions::new(options);
+                // let options = WithOptionsSecResolved::without_secrets(options);
+                let options = WithOptionsSecResolved::without_secrets(options);
                 let format_desc = (&options).try_into().unwrap();
                 match plan_root.gen_sink_plan(
                     sink_name.to_string(),
diff --git a/src/frontend/planner_test/tests/testdata/input/agg.yaml b/src/frontend/planner_test/tests/testdata/input/agg.yaml
index 70f83549ff18c..25f62054f1e66 100644
--- a/src/frontend/planner_test/tests/testdata/input/agg.yaml
+++ b/src/frontend/planner_test/tests/testdata/input/agg.yaml
@@ -1000,3 +1000,59 @@
   expected_outputs:
     - batch_plan
     - stream_plan
+- name: test duplicate agg
+  sql: |
+    CREATE TABLE t (v1 int);
+    SELECT sum(v1) as x, count(v1) as y, sum(v1) as z, count(v1) as w from t;
+  expected_outputs:
+    - logical_plan
+    - stream_plan
+- name: test simple approx_percentile alone
+  sql: |
+    CREATE TABLE t (v1 int);
+    SELECT approx_percentile(0.5, 0.01) WITHIN GROUP (order by v1) from t;
+  expected_outputs:
+    - logical_plan
+    - stream_plan
+- name: test simple approx_percentile with other simple aggs
+  sql: |
+    CREATE TABLE t (v1 int);
+    SELECT approx_percentile(0.5, 0.01) WITHIN GROUP (order by v1), sum(v1) from t;
+  expected_outputs:
+    - logical_plan
+    - stream_plan
+- name: test simple approx_percentile with other simple aggs (sum, count)
+  sql: |
+    CREATE TABLE t (v1 int);
+    SELECT sum(v1) as s1, approx_percentile(0.5, 0.01) WITHIN GROUP (order by v1), sum(v1) as s2, count(v1) from t;
+  expected_outputs:
+    - logical_plan
+    - stream_plan
+- name: test simple approx_percentile with duplicate approx_percentile
+  sql: |
+    CREATE TABLE t (v1 int);
+    SELECT approx_percentile(0.5, 0.01) WITHIN GROUP (order by v1) as x, approx_percentile(0.5, 0.01) WITHIN GROUP (order by v1) as y from t;
+  expected_outputs:
+    - logical_plan
+    - stream_plan
+- name: test simple approx_percentile with different approx_percentile
+  sql: |
+    CREATE TABLE t (v1 int, v2 int);
+    SELECT approx_percentile(0.5, 0.01) WITHIN GROUP (order by v1) as x, approx_percentile(0.5, 0.01) WITHIN GROUP (order by v2) as y from t;
+  expected_outputs:
+    - logical_plan
+    - stream_plan
+- name: test simple approx_percentile with different approx_percentile interleaved with stateless simple aggs
+  sql: |
+    CREATE TABLE t (v1 int, v2 int);
+    SELECT sum(v1) as s1, approx_percentile(0.5, 0.01) WITHIN GROUP (order by v1) as x, count(*), sum(v2) as s2, approx_percentile(0.5, 0.01) WITHIN GROUP (order by v2) as y from t;
+  expected_outputs:
+    - logical_plan
+    - stream_plan
+- name: test simple approx_percentile with descending order
+  sql: |
+    CREATE TABLE t (v1 int, v2 int);
+    SELECT sum(v1) as s1, approx_percentile(0.2, 0.01) WITHIN GROUP (order by v1 desc) from t;
+  expected_outputs:
+    - logical_plan
+    - stream_plan
\ No newline at end of file
diff --git a/src/frontend/planner_test/tests/testdata/input/generated_columns.yaml b/src/frontend/planner_test/tests/testdata/input/generated_columns.yaml
index bd6a8f453ab71..7078f6e797342 100644
--- a/src/frontend/planner_test/tests/testdata/input/generated_columns.yaml
+++ b/src/frontend/planner_test/tests/testdata/input/generated_columns.yaml
@@ -14,6 +14,11 @@
     select proctime();
   expected_outputs:
   - binder_error
+- name: proctime cast to with timezone
+  sql: |
+    explain create table t1 (proc_time TIMESTAMPTZ AS proctime());
+  expected_outputs:
+  - explain_output
 - name: proctime cast to without timezone
   sql: |
     explain create table t1 (proc_time TIMESTAMP AS proctime());
diff --git a/src/frontend/planner_test/tests/testdata/input/temporal_filter.yaml b/src/frontend/planner_test/tests/testdata/input/temporal_filter.yaml
index 6bd62c1ce4d61..ce8fc00e15496 100644
--- a/src/frontend/planner_test/tests/testdata/input/temporal_filter.yaml
+++ b/src/frontend/planner_test/tests/testdata/input/temporal_filter.yaml
@@ -3,96 +3,96 @@
     create table t1 (ts timestamp with time zone);
     select * from t1 where ts + interval '1 hour' > now();
   expected_outputs:
-  - stream_plan
+    - stream_plan
 - name: Temporal filter works on complex columns on LHS (part 2)
   sql: |
     create table t1 (ts timestamp with time zone, time_to_live interval);
     select * from t1 where ts + time_to_live * 1.5 > now();
   expected_outputs:
-  - stream_plan
+    - stream_plan
 - name: Temporal filter works on complex columns on LHS (part 2, flipped)
   sql: |
     create table t1 (ts timestamp with time zone, additional_time_to_live interval);
     select * from t1 where now() - interval '15 minutes' < ts + additional_time_to_live * 1.5;
   expected_outputs:
-  - stream_plan
+    - stream_plan
 - name: Temporal filter with `now()` in upper bound
   sql: |-
     create table t1 (ts timestamp with time zone);
     select * from t1 where now() - interval '15 minutes' > ts;
   expected_outputs:
-  - stream_plan
-  - stream_dist_plan
+    - stream_plan
+    - stream_dist_plan
 - name: Temporal filter with equal condition
   sql: |-
     create table t1 (ts timestamp with time zone);
-    select * from t1 where date_trunc('week', now())  = date_trunc('week',ts);
+    select * from t1 where date_trunc('week', now()) = date_trunc('week',ts);
   expected_outputs:
-  - stream_plan
-  - stream_dist_plan
+    - stream_plan
+    - stream_dist_plan
 - name: Temporal filter with `now()` in upper bound on append only table
   sql: |-
     create table t1 (ts timestamp with time zone) APPEND ONLY;
     select * from t1 where now() - interval '15 minutes' > ts;
   expected_outputs:
-  - stream_plan
-  - stream_dist_plan
+    - stream_plan
+    - stream_dist_plan
 - name: Temporal filter reorders now expressions correctly
   sql: |
     create table t1 (ts timestamp with time zone);
     select * from t1 where ts < now() - interval '1 hour' and ts >= now() - interval '2 hour';
   expected_outputs:
-  - stream_plan
-  - stream_dist_plan
+    - stream_plan
+    - stream_dist_plan
 - name: Temporal filter in on clause for inner join's left side
   sql: |
     create table t1 (a int, ta timestamp with time zone);
     create table t2 (b int, tb timestamp with time zone);
     select * from t1 join t2 on a = b AND ta < now() - interval '1 hour' and ta >= now() - interval '2 hour';
   expected_outputs:
-  - stream_plan
+    - stream_plan
 - name: Temporal filter in on clause for left join's left side
   sql: |
     create table t1 (a int, ta timestamp with time zone);
     create table t2 (b int, tb timestamp with time zone);
     select * from t1 left join t2 on a = b AND ta < now() - interval '1 hour' and ta >= now() - interval '2 hour';
   expected_outputs:
-  - stream_error
+    - stream_error
 - name: Temporal filter in on clause for right join's left side
   sql: |
     create table t1 (a int, ta timestamp with time zone);
     create table t2 (b int, tb timestamp with time zone);
     select * from t1 right join t2 on a = b AND ta < now() - interval '1 hour' and ta >= now() - interval '2 hour';
   expected_outputs:
-  - stream_plan
+    - stream_plan
 - name: Temporal filter in on clause for full join's left side
   sql: |
     create table t1 (a int, ta timestamp with time zone);
     create table t2 (b int, tb timestamp with time zone);
     select * from t1 full join t2 on a = b AND ta < now() - interval '1 hour' and ta >= now() - interval '2 hour';
   expected_outputs:
-  - stream_error
+    - stream_error
 - name: Temporal filter in on clause for left join's right side
   sql: |
     create table t1 (a int, ta timestamp with time zone);
     create table t2 (b int, tb timestamp with time zone);
     select * from t1 left join t2 on a = b AND tb < now() - interval '1 hour' and tb >= now() - interval '2 hour';
   expected_outputs:
-  - stream_plan
+    - stream_plan
 - name: Temporal filter in on clause for right join's right side
   sql: |
     create table t1 (a int, ta timestamp with time zone);
     create table t2 (b int, tb timestamp with time zone);
     select * from t1 right join t2 on a = b AND tb < now() - interval '1 hour' and tb >= now() - interval '2 hour';
   expected_outputs:
-  - stream_error
+    - stream_error
 - name: Temporal filter after temporal join
   sql: |
     create table stream(id1 int, a1 int, b1 int, v1 timestamp with time zone) APPEND ONLY;
     create table version(id2 int, a2 int, b2 int, primary key (id2));
     select id1, a1, id2, v1 from stream left join version FOR SYSTEM_TIME AS OF PROCTIME() on id1 = id2 where v1 > now();
   expected_outputs:
-  - stream_plan
+    - stream_plan
 - name: Temporal filter with or predicate
   sql: |
     create table t1 (ts timestamp with time zone);
@@ -116,4 +116,22 @@
     create table t (t timestamp with time zone, a int);
     select * from t where (t > NOW() - INTERVAL '1 hour' OR t is NULL OR a < 1) AND (t < NOW() - INTERVAL '1 hour' OR a > 1);
   expected_outputs:
-    - stream_plan
\ No newline at end of file
+    - stream_plan
+- name: Non-trivial now expression
+  sql: |
+    create table t (ts timestamp with time zone, a int);
+    select * from t where ts + interval '1 hour' > date_trunc('day', now());
+  expected_outputs:
+    - stream_plan
+- name: Non-trivial now expression 2
+  sql: |
+    create table t (ts timestamp with time zone, a int);
+    select * from t where ts + interval '1 hour' > date_trunc('day', ('2024-07-18 00:00:00+00:00'::timestamptz - ('2024-07-18 00:00:00+00:00'::timestamptz - now())));
+  expected_outputs:
+    - stream_plan
+- name: Non-monotonic now expression
+  sql: |
+    create table t (ts timestamp with time zone, a int);
+    select * from t where a > extract(hour from now());
+  expected_outputs:
+    - stream_error
diff --git a/src/frontend/planner_test/tests/testdata/input/union.yaml b/src/frontend/planner_test/tests/testdata/input/union.yaml
index 8775d4f9d36f2..93e6b00089066 100644
--- a/src/frontend/planner_test/tests/testdata/input/union.yaml
+++ b/src/frontend/planner_test/tests/testdata/input/union.yaml
@@ -95,3 +95,31 @@
     select * from t1 union all select * from t2 union all select * from t3 union all select * from t4 union all select * from t5;
   expected_outputs:
     - stream_dist_plan
+
+- name: test corresponding union
+  sql: |
+    create table t1 (a int, b numeric, c bigint);
+    create table t2 (a int, b numeric, y bigint);
+    create table t3 (x int, b numeric, c bigint);
+    select * from t1 union corresponding select * from t2 union all corresponding by (b) select * from t3;
+  expected_outputs:
+    - batch_plan
+    - stream_plan
+    - stream_dist_plan
+
+- name: test corresponding union error - corresponding list
+  sql: |
+    create table t1 (a int, b numeric, c bigint);
+    create table t2 (a int, b numeric, y bigint);
+    create table t3 (x int, b numeric, c bigint);
+    select * from t1 union corresponding select * from t2 union all corresponding by (c) select * from t3;
+  expected_outputs:
+    - binder_error
+
+- name: test corresponding union error - duplicate names
+  sql: |
+    create table t1 (a int, b numeric, c bigint);
+    create table t2 (a int, b numeric, y bigint);
+    select a, b as a from t1 union corresponding select * from t2;
+  expected_outputs:
+    - binder_error
\ No newline at end of file
diff --git a/src/frontend/planner_test/tests/testdata/output/agg.yaml b/src/frontend/planner_test/tests/testdata/output/agg.yaml
index 4c75b83187741..b6c692fd47364 100644
--- a/src/frontend/planner_test/tests/testdata/output/agg.yaml
+++ b/src/frontend/planner_test/tests/testdata/output/agg.yaml
@@ -1863,3 +1863,180 @@
             └─StreamHashAgg { group_key: [t.a, t.b], aggs: [sum(t.c), sum(t.d), count(t.d), max(t.e), count] }
               └─StreamExchange { dist: HashShard(t.a, t.b) }
                 └─StreamTableScan { table: t, columns: [t.a, t.b, t.c, t.d, t.e, t._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t._row_id], pk: [_row_id], dist: UpstreamHashShard(t._row_id) }
+- name: test duplicate agg
+  sql: |
+    CREATE TABLE t (v1 int);
+    SELECT sum(v1) as x, count(v1) as y, sum(v1) as z, count(v1) as w from t;
+  logical_plan: |-
+    LogicalProject { exprs: [sum(t.v1), count(t.v1), sum(t.v1), count(t.v1)] }
+    └─LogicalAgg { aggs: [sum(t.v1), count(t.v1)] }
+      └─LogicalProject { exprs: [t.v1] }
+        └─LogicalScan { table: t, columns: [t.v1, t._row_id] }
+  stream_plan: |-
+    StreamMaterialize { columns: [x, y, z, w], stream_key: [], pk_columns: [], pk_conflict: NoCheck }
+    └─StreamProject { exprs: [sum(sum(t.v1)), sum0(count(t.v1)), sum(sum(t.v1)), sum0(count(t.v1))] }
+      └─StreamSimpleAgg { aggs: [sum(sum(t.v1)), sum0(count(t.v1)), count] }
+        └─StreamExchange { dist: Single }
+          └─StreamStatelessSimpleAgg { aggs: [sum(t.v1), count(t.v1)] }
+            └─StreamTableScan { table: t, columns: [t.v1, t._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t._row_id], pk: [_row_id], dist: UpstreamHashShard(t._row_id) }
+- name: test simple approx_percentile alone
+  sql: |
+    CREATE TABLE t (v1 int);
+    SELECT approx_percentile(0.5, 0.01) WITHIN GROUP (order by v1) from t;
+  logical_plan: |-
+    LogicalProject { exprs: [approx_percentile($expr1)] }
+    └─LogicalAgg { aggs: [approx_percentile($expr1)] }
+      └─LogicalProject { exprs: [t.v1::Float64 as $expr1] }
+        └─LogicalScan { table: t, columns: [t.v1, t._row_id] }
+  stream_plan: |-
+    StreamMaterialize { columns: [approx_percentile], stream_key: [], pk_columns: [], pk_conflict: NoCheck }
+    └─StreamGlobalApproxPercentile { quantile: 0.5:Float64, relative_error: 0.01:Float64 }
+      └─StreamExchange { dist: Single }
+        └─StreamLocalApproxPercentile { percentile_col: $expr1, quantile: 0.5:Float64, relative_error: 0.01:Float64 }
+          └─StreamProject { exprs: [t.v1::Float64 as $expr1, t._row_id] }
+            └─StreamTableScan { table: t, columns: [t.v1, t._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t._row_id], pk: [_row_id], dist: UpstreamHashShard(t._row_id) }
+- name: test simple approx_percentile with other simple aggs
+  sql: |
+    CREATE TABLE t (v1 int);
+    SELECT approx_percentile(0.5, 0.01) WITHIN GROUP (order by v1), sum(v1) from t;
+  logical_plan: |-
+    LogicalProject { exprs: [approx_percentile($expr1), sum(t.v1)] }
+    └─LogicalAgg { aggs: [approx_percentile($expr1), sum(t.v1)] }
+      └─LogicalProject { exprs: [t.v1::Float64 as $expr1, t.v1] }
+        └─LogicalScan { table: t, columns: [t.v1, t._row_id] }
+  stream_plan: |-
+    StreamMaterialize { columns: [approx_percentile, sum], stream_key: [], pk_columns: [], pk_conflict: NoCheck }
+    └─StreamRowMerge { output: [approx_percentile:Float64, sum(sum(t.v1)):Int64] }
+      ├─StreamGlobalApproxPercentile { quantile: 0.5:Float64, relative_error: 0.01:Float64 }
+      │ └─StreamExchange { dist: Single }
+      │   └─StreamLocalApproxPercentile { percentile_col: $expr1, quantile: 0.5:Float64, relative_error: 0.01:Float64 }
+      │     └─StreamShare { id: 2 }
+      │       └─StreamProject { exprs: [t.v1::Float64 as $expr1, t.v1, t._row_id] }
+      │         └─StreamTableScan { table: t, columns: [t.v1, t._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t._row_id], pk: [_row_id], dist: UpstreamHashShard(t._row_id) }
+      └─StreamSimpleAgg { aggs: [sum(sum(t.v1)), count] }
+        └─StreamExchange { dist: Single }
+          └─StreamStatelessSimpleAgg { aggs: [sum(t.v1)] }
+            └─StreamShare { id: 2 }
+              └─StreamProject { exprs: [t.v1::Float64 as $expr1, t.v1, t._row_id] }
+                └─StreamTableScan { table: t, columns: [t.v1, t._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t._row_id], pk: [_row_id], dist: UpstreamHashShard(t._row_id) }
+- name: test simple approx_percentile with other simple aggs (sum, count)
+  sql: |
+    CREATE TABLE t (v1 int);
+    SELECT sum(v1) as s1, approx_percentile(0.5, 0.01) WITHIN GROUP (order by v1), sum(v1) as s2, count(v1) from t;
+  logical_plan: |-
+    LogicalProject { exprs: [sum(t.v1), approx_percentile($expr1), sum(t.v1), count(t.v1)] }
+    └─LogicalAgg { aggs: [sum(t.v1), approx_percentile($expr1), count(t.v1)] }
+      └─LogicalProject { exprs: [t.v1, t.v1::Float64 as $expr1] }
+        └─LogicalScan { table: t, columns: [t.v1, t._row_id] }
+  stream_plan: |-
+    StreamMaterialize { columns: [s1, approx_percentile, s2, count], stream_key: [], pk_columns: [], pk_conflict: NoCheck }
+    └─StreamProject { exprs: [sum(sum(t.v1)), approx_percentile, sum(sum(t.v1)), sum0(count(t.v1))] }
+      └─StreamRowMerge { output: [sum(sum(t.v1)):Int64, approx_percentile:Float64, sum0(count(t.v1)):Int64] }
+        ├─StreamGlobalApproxPercentile { quantile: 0.5:Float64, relative_error: 0.01:Float64 }
+        │ └─StreamExchange { dist: Single }
+        │   └─StreamLocalApproxPercentile { percentile_col: $expr1, quantile: 0.5:Float64, relative_error: 0.01:Float64 }
+        │     └─StreamShare { id: 2 }
+        │       └─StreamProject { exprs: [t.v1, t.v1::Float64 as $expr1, t._row_id] }
+        │         └─StreamTableScan { table: t, columns: [t.v1, t._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t._row_id], pk: [_row_id], dist: UpstreamHashShard(t._row_id) }
+        └─StreamSimpleAgg { aggs: [sum(sum(t.v1)), sum0(count(t.v1)), count] }
+          └─StreamExchange { dist: Single }
+            └─StreamStatelessSimpleAgg { aggs: [sum(t.v1), count(t.v1)] }
+              └─StreamShare { id: 2 }
+                └─StreamProject { exprs: [t.v1, t.v1::Float64 as $expr1, t._row_id] }
+                  └─StreamTableScan { table: t, columns: [t.v1, t._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t._row_id], pk: [_row_id], dist: UpstreamHashShard(t._row_id) }
+- name: test simple approx_percentile with duplicate approx_percentile
+  sql: |
+    CREATE TABLE t (v1 int);
+    SELECT approx_percentile(0.5, 0.01) WITHIN GROUP (order by v1) as x, approx_percentile(0.5, 0.01) WITHIN GROUP (order by v1) as y from t;
+  logical_plan: |-
+    LogicalProject { exprs: [approx_percentile($expr1), approx_percentile($expr1)] }
+    └─LogicalAgg { aggs: [approx_percentile($expr1)] }
+      └─LogicalProject { exprs: [t.v1::Float64 as $expr1] }
+        └─LogicalScan { table: t, columns: [t.v1, t._row_id] }
+  stream_plan: |-
+    StreamMaterialize { columns: [x, y], stream_key: [], pk_columns: [], pk_conflict: NoCheck }
+    └─StreamProject { exprs: [approx_percentile, approx_percentile] }
+      └─StreamGlobalApproxPercentile { quantile: 0.5:Float64, relative_error: 0.01:Float64 }
+        └─StreamExchange { dist: Single }
+          └─StreamLocalApproxPercentile { percentile_col: $expr1, quantile: 0.5:Float64, relative_error: 0.01:Float64 }
+            └─StreamProject { exprs: [t.v1::Float64 as $expr1, t._row_id] }
+              └─StreamTableScan { table: t, columns: [t.v1, t._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t._row_id], pk: [_row_id], dist: UpstreamHashShard(t._row_id) }
+- name: test simple approx_percentile with different approx_percentile
+  sql: |
+    CREATE TABLE t (v1 int, v2 int);
+    SELECT approx_percentile(0.5, 0.01) WITHIN GROUP (order by v1) as x, approx_percentile(0.5, 0.01) WITHIN GROUP (order by v2) as y from t;
+  logical_plan: |-
+    LogicalProject { exprs: [approx_percentile($expr1), approx_percentile($expr2)] }
+    └─LogicalAgg { aggs: [approx_percentile($expr1), approx_percentile($expr2)] }
+      └─LogicalProject { exprs: [t.v1::Float64 as $expr1, t.v2::Float64 as $expr2] }
+        └─LogicalScan { table: t, columns: [t.v1, t.v2, t._row_id] }
+  stream_plan: |-
+    StreamMaterialize { columns: [x, y], stream_key: [], pk_columns: [], pk_conflict: NoCheck }
+    └─StreamRowMerge { output: [approx_percentile:Float64, approx_percentile:Float64] }
+      ├─StreamGlobalApproxPercentile { quantile: 0.5:Float64, relative_error: 0.01:Float64 }
+      │ └─StreamExchange { dist: Single }
+      │   └─StreamLocalApproxPercentile { percentile_col: $expr1, quantile: 0.5:Float64, relative_error: 0.01:Float64 }
+      │     └─StreamShare { id: 2 }
+      │       └─StreamProject { exprs: [t.v1::Float64 as $expr1, t.v2::Float64 as $expr2, t._row_id] }
+      │         └─StreamTableScan { table: t, columns: [t.v1, t.v2, t._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t._row_id], pk: [_row_id], dist: UpstreamHashShard(t._row_id) }
+      └─StreamGlobalApproxPercentile { quantile: 0.5:Float64, relative_error: 0.01:Float64 }
+        └─StreamExchange { dist: Single }
+          └─StreamLocalApproxPercentile { percentile_col: $expr2, quantile: 0.5:Float64, relative_error: 0.01:Float64 }
+            └─StreamShare { id: 2 }
+              └─StreamProject { exprs: [t.v1::Float64 as $expr1, t.v2::Float64 as $expr2, t._row_id] }
+                └─StreamTableScan { table: t, columns: [t.v1, t.v2, t._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t._row_id], pk: [_row_id], dist: UpstreamHashShard(t._row_id) }
+- name: test simple approx_percentile with different approx_percentile interleaved with stateless simple aggs
+  sql: |
+    CREATE TABLE t (v1 int, v2 int);
+    SELECT sum(v1) as s1, approx_percentile(0.5, 0.01) WITHIN GROUP (order by v1) as x, count(*), sum(v2) as s2, approx_percentile(0.5, 0.01) WITHIN GROUP (order by v2) as y from t;
+  logical_plan: |-
+    LogicalProject { exprs: [sum(t.v1), approx_percentile($expr1), count, sum(t.v2), approx_percentile($expr2)] }
+    └─LogicalAgg { aggs: [sum(t.v1), approx_percentile($expr1), count, sum(t.v2), approx_percentile($expr2)] }
+      └─LogicalProject { exprs: [t.v1, t.v1::Float64 as $expr1, t.v2, t.v2::Float64 as $expr2] }
+        └─LogicalScan { table: t, columns: [t.v1, t.v2, t._row_id] }
+  stream_plan: |-
+    StreamMaterialize { columns: [s1, x, count, s2, y], stream_key: [], pk_columns: [], pk_conflict: NoCheck }
+    └─StreamRowMerge { output: [sum(sum(t.v1)):Int64, approx_percentile:Float64, sum0(count):Int64, sum(sum(t.v2)):Int64, approx_percentile:Float64] }
+      ├─StreamRowMerge { output: [approx_percentile:Float64, approx_percentile:Float64] }
+      │ ├─StreamGlobalApproxPercentile { quantile: 0.5:Float64, relative_error: 0.01:Float64 }
+      │ │ └─StreamExchange { dist: Single }
+      │ │   └─StreamLocalApproxPercentile { percentile_col: $expr1, quantile: 0.5:Float64, relative_error: 0.01:Float64 }
+      │ │     └─StreamShare { id: 2 }
+      │ │       └─StreamProject { exprs: [t.v1, t.v1::Float64 as $expr1, t.v2, t.v2::Float64 as $expr2, t._row_id] }
+      │ │         └─StreamTableScan { table: t, columns: [t.v1, t.v2, t._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t._row_id], pk: [_row_id], dist: UpstreamHashShard(t._row_id) }
+      │ └─StreamGlobalApproxPercentile { quantile: 0.5:Float64, relative_error: 0.01:Float64 }
+      │   └─StreamExchange { dist: Single }
+      │     └─StreamLocalApproxPercentile { percentile_col: $expr2, quantile: 0.5:Float64, relative_error: 0.01:Float64 }
+      │       └─StreamShare { id: 2 }
+      │         └─StreamProject { exprs: [t.v1, t.v1::Float64 as $expr1, t.v2, t.v2::Float64 as $expr2, t._row_id] }
+      │           └─StreamTableScan { table: t, columns: [t.v1, t.v2, t._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t._row_id], pk: [_row_id], dist: UpstreamHashShard(t._row_id) }
+      └─StreamSimpleAgg { aggs: [sum(sum(t.v1)), sum0(count), sum(sum(t.v2)), count] }
+        └─StreamExchange { dist: Single }
+          └─StreamStatelessSimpleAgg { aggs: [sum(t.v1), count, sum(t.v2)] }
+            └─StreamShare { id: 2 }
+              └─StreamProject { exprs: [t.v1, t.v1::Float64 as $expr1, t.v2, t.v2::Float64 as $expr2, t._row_id] }
+                └─StreamTableScan { table: t, columns: [t.v1, t.v2, t._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t._row_id], pk: [_row_id], dist: UpstreamHashShard(t._row_id) }
+- name: test simple approx_percentile with descending order
+  sql: |
+    CREATE TABLE t (v1 int, v2 int);
+    SELECT sum(v1) as s1, approx_percentile(0.2, 0.01) WITHIN GROUP (order by v1 desc) from t;
+  logical_plan: |-
+    LogicalProject { exprs: [sum(t.v1), approx_percentile($expr1)] }
+    └─LogicalAgg { aggs: [sum(t.v1), approx_percentile($expr1)] }
+      └─LogicalProject { exprs: [t.v1, t.v1::Float64 as $expr1] }
+        └─LogicalScan { table: t, columns: [t.v1, t.v2, t._row_id] }
+  stream_plan: |-
+    StreamMaterialize { columns: [s1, approx_percentile], stream_key: [], pk_columns: [], pk_conflict: NoCheck }
+    └─StreamRowMerge { output: [sum(sum(t.v1)):Int64, approx_percentile:Float64] }
+      ├─StreamGlobalApproxPercentile { quantile: 0.8:Float64, relative_error: 0.01:Float64 }
+      │ └─StreamExchange { dist: Single }
+      │   └─StreamLocalApproxPercentile { percentile_col: $expr1, quantile: 0.8:Float64, relative_error: 0.01:Float64 }
+      │     └─StreamShare { id: 2 }
+      │       └─StreamProject { exprs: [t.v1, t.v1::Float64 as $expr1, t._row_id] }
+      │         └─StreamTableScan { table: t, columns: [t.v1, t._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t._row_id], pk: [_row_id], dist: UpstreamHashShard(t._row_id) }
+      └─StreamSimpleAgg { aggs: [sum(sum(t.v1)), count] }
+        └─StreamExchange { dist: Single }
+          └─StreamStatelessSimpleAgg { aggs: [sum(t.v1)] }
+            └─StreamShare { id: 2 }
+              └─StreamProject { exprs: [t.v1, t.v1::Float64 as $expr1, t._row_id] }
+                └─StreamTableScan { table: t, columns: [t.v1, t._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t._row_id], pk: [_row_id], dist: UpstreamHashShard(t._row_id) }
diff --git a/src/frontend/planner_test/tests/testdata/output/expr.yaml b/src/frontend/planner_test/tests/testdata/output/expr.yaml
index f88f7c4d69b76..d24d0f0eeba18 100644
--- a/src/frontend/planner_test/tests/testdata/output/expr.yaml
+++ b/src/frontend/planner_test/tests/testdata/output/expr.yaml
@@ -520,7 +520,7 @@
   sql: |
     create table t (v1 timestamp with time zone, v2 timestamp with time zone);
     select * from t where v1 >= now() or v2 >= now();
-  stream_error: Conditions containing now must be of the form `input_expr cmp now() [+- const_expr]` or `now() [+- const_expr] cmp input_expr`, where `input_expr` references a column and contains no `now()`.
+  stream_error: Conditions containing now must be in the form of `input_expr cmp now_expr` or `now_expr cmp input_expr`, where `input_expr` references a column and contains no `now()`, and `now_expr` is a non-decreasing expression contains `now()`.
 - name: now inside HAVING clause
   sql: |
     create table t (v1 timestamp with time zone, v2 int);
diff --git a/src/frontend/planner_test/tests/testdata/output/generated_columns.yaml b/src/frontend/planner_test/tests/testdata/output/generated_columns.yaml
index 3ed95c1ac1463..92f6aaa47e175 100644
--- a/src/frontend/planner_test/tests/testdata/output/generated_columns.yaml
+++ b/src/frontend/planner_test/tests/testdata/output/generated_columns.yaml
@@ -26,6 +26,17 @@
 
     Caused by:
       Invalid input syntax: Function `PROCTIME()` is only allowed in CREATE TABLE/SOURCE. Is `NOW()` what you want?
+- name: proctime cast to with timezone
+  sql: |
+    explain create table t1 (proc_time TIMESTAMPTZ AS proctime());
+  explain_output: |
+    StreamMaterialize { columns: [proc_time, _row_id(hidden)], stream_key: [_row_id], pk_columns: [_row_id], pk_conflict: Overwrite, watermark_columns: [proc_time] }
+    └─StreamRowIdGen { row_id_index: 1 }
+      └─StreamUnion { all: true, output_watermarks: [$expr1] }
+        └─StreamExchange { dist: HashShard(_row_id) }
+          └─StreamProject { exprs: [Proctime as $expr1, _row_id], output_watermarks: [$expr1] }
+            └─StreamDml { columns: [_row_id] }
+              └─StreamSource
 - name: proctime cast to without timezone
   sql: |
     explain create table t1 (proc_time TIMESTAMP AS proctime());
diff --git a/src/frontend/planner_test/tests/testdata/output/subquery.yaml b/src/frontend/planner_test/tests/testdata/output/subquery.yaml
index e113a0aca4d1d..98a0fcbf8fdd3 100644
--- a/src/frontend/planner_test/tests/testdata/output/subquery.yaml
+++ b/src/frontend/planner_test/tests/testdata/output/subquery.yaml
@@ -237,7 +237,7 @@
         │       │ │ │ │ ├─LogicalUnion { all: true }
         │       │ │ │ │ │ ├─LogicalUnion { all: true }
         │       │ │ │ │ │ │ ├─LogicalProject { exprs: [rw_tables.id, rw_tables.name, 'table':Varchar, rw_tables.schema_id, rw_tables.owner, rw_tables.definition, rw_tables.acl] }
-        │       │ │ │ │ │ │ │ └─LogicalSysScan { table: rw_tables, columns: [rw_tables.id, rw_tables.name, rw_tables.schema_id, rw_tables.owner, rw_tables.definition, rw_tables.acl, rw_tables.initialized_at, rw_tables.created_at, rw_tables.initialized_at_cluster_version, rw_tables.created_at_cluster_version] }
+        │       │ │ │ │ │ │ │ └─LogicalSysScan { table: rw_tables, columns: [rw_tables.id, rw_tables.name, rw_tables.schema_id, rw_tables.owner, rw_tables.definition, rw_tables.append_only, rw_tables.acl, rw_tables.initialized_at, rw_tables.created_at, rw_tables.initialized_at_cluster_version, rw_tables.created_at_cluster_version] }
         │       │ │ │ │ │ │ └─LogicalProject { exprs: [rw_system_tables.id, rw_system_tables.name, 'system table':Varchar, rw_system_tables.schema_id, rw_system_tables.owner, rw_system_tables.definition, rw_system_tables.acl] }
         │       │ │ │ │ │ │   └─LogicalSysScan { table: rw_system_tables, columns: [rw_system_tables.id, rw_system_tables.name, rw_system_tables.schema_id, rw_system_tables.owner, rw_system_tables.definition, rw_system_tables.acl] }
         │       │ │ │ │ │ └─LogicalProject { exprs: [rw_sources.id, rw_sources.name, 'source':Varchar, rw_sources.schema_id, rw_sources.owner, rw_sources.definition, rw_sources.acl] }
@@ -249,7 +249,7 @@
         │       │ │ └─LogicalProject { exprs: [rw_subscriptions.id, rw_subscriptions.name, 'subscription':Varchar, rw_subscriptions.schema_id, rw_subscriptions.owner, rw_subscriptions.definition, rw_subscriptions.acl] }
         │       │ │   └─LogicalSysScan { table: rw_subscriptions, columns: [rw_subscriptions.id, rw_subscriptions.name, rw_subscriptions.schema_id, rw_subscriptions.owner, rw_subscriptions.definition, rw_subscriptions.acl, rw_subscriptions.initialized_at, rw_subscriptions.created_at, rw_subscriptions.initialized_at_cluster_version, rw_subscriptions.created_at_cluster_version] }
         │       │ └─LogicalProject { exprs: [rw_materialized_views.id, rw_materialized_views.name, 'materialized view':Varchar, rw_materialized_views.schema_id, rw_materialized_views.owner, rw_materialized_views.definition, rw_materialized_views.acl] }
-        │       │   └─LogicalSysScan { table: rw_materialized_views, columns: [rw_materialized_views.id, rw_materialized_views.name, rw_materialized_views.schema_id, rw_materialized_views.owner, rw_materialized_views.definition, rw_materialized_views.acl, rw_materialized_views.initialized_at, rw_materialized_views.created_at, rw_materialized_views.initialized_at_cluster_version, rw_materialized_views.created_at_cluster_version] }
+        │       │   └─LogicalSysScan { table: rw_materialized_views, columns: [rw_materialized_views.id, rw_materialized_views.name, rw_materialized_views.schema_id, rw_materialized_views.owner, rw_materialized_views.definition, rw_materialized_views.append_only, rw_materialized_views.acl, rw_materialized_views.initialized_at, rw_materialized_views.created_at, rw_materialized_views.initialized_at_cluster_version, rw_materialized_views.created_at_cluster_version] }
         │       └─LogicalProject { exprs: [rw_views.id, rw_views.name, 'view':Varchar, rw_views.schema_id, rw_views.owner, rw_views.definition, rw_views.acl] }
         │         └─LogicalSysScan { table: rw_views, columns: [rw_views.id, rw_views.name, rw_views.schema_id, rw_views.owner, rw_views.definition, rw_views.acl] }
         └─LogicalShare { id: 20 }
diff --git a/src/frontend/planner_test/tests/testdata/output/temporal_filter.yaml b/src/frontend/planner_test/tests/testdata/output/temporal_filter.yaml
index 7bbd43ce3c35c..514a56f7dff6d 100644
--- a/src/frontend/planner_test/tests/testdata/output/temporal_filter.yaml
+++ b/src/frontend/planner_test/tests/testdata/output/temporal_filter.yaml
@@ -42,7 +42,7 @@
     select * from t1 where now() - interval '15 minutes' > ts;
   stream_plan: |-
     StreamMaterialize { columns: [ts, t1._row_id(hidden)], stream_key: [t1._row_id], pk_columns: [t1._row_id], pk_conflict: NoCheck }
-    └─StreamDynamicFilter { predicate: (t1.ts < $expr1), output: [t1.ts, t1._row_id], condition_always_relax: true }
+    └─StreamDynamicFilter { predicate: (t1.ts < $expr1), output: [t1.ts, t1._row_id], cleaned_by_watermark: true }
       ├─StreamTableScan { table: t1, columns: [t1.ts, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
       └─StreamExchange { dist: Broadcast }
         └─StreamProject { exprs: [SubtractWithTimeZone(now, '00:15:00':Interval, 'UTC':Varchar) as $expr1], output_watermarks: [$expr1] }
@@ -50,8 +50,7 @@
   stream_dist_plan: |+
     Fragment 0
     StreamMaterialize { columns: [ts, t1._row_id(hidden)], stream_key: [t1._row_id], pk_columns: [t1._row_id], pk_conflict: NoCheck } { tables: [ Materialize: 4294967294 ] }
-    └── StreamDynamicFilter { predicate: (t1.ts < $expr1), output: [t1.ts, t1._row_id], condition_always_relax: true }
-        ├── tables: [ DynamicFilterLeftNotSatisfy: 0, DynamicFilterRight: 1 ]
+    └── StreamDynamicFilter { predicate: (t1.ts < $expr1), output: [t1.ts, t1._row_id], cleaned_by_watermark: true } { tables: [ DynamicFilterLeft: 0, DynamicFilterRight: 1 ] }
         ├── StreamTableScan { table: t1, columns: [t1.ts, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
         │   ├── tables: [ StreamScan: 2 ]
         │   ├── Upstream
@@ -81,7 +80,7 @@
 - name: Temporal filter with equal condition
   sql: |-
     create table t1 (ts timestamp with time zone);
-    select * from t1 where date_trunc('week', now())  = date_trunc('week',ts);
+    select * from t1 where date_trunc('week', now()) = date_trunc('week',ts);
   stream_plan: |-
     StreamMaterialize { columns: [ts, t1._row_id(hidden), $expr1(hidden)], stream_key: [t1._row_id, $expr1], pk_columns: [t1._row_id, $expr1], pk_conflict: NoCheck }
     └─StreamExchange { dist: HashShard(t1._row_id, $expr1) }
@@ -141,7 +140,7 @@
     select * from t1 where now() - interval '15 minutes' > ts;
   stream_plan: |-
     StreamMaterialize { columns: [ts, t1._row_id(hidden)], stream_key: [t1._row_id], pk_columns: [t1._row_id], pk_conflict: NoCheck }
-    └─StreamDynamicFilter [append_only] { predicate: (t1.ts < $expr1), output: [t1.ts, t1._row_id], condition_always_relax: true }
+    └─StreamDynamicFilter [append_only] { predicate: (t1.ts < $expr1), output: [t1.ts, t1._row_id], cleaned_by_watermark: true }
       ├─StreamTableScan { table: t1, columns: [t1.ts, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
       └─StreamExchange { dist: Broadcast }
         └─StreamProject { exprs: [SubtractWithTimeZone(now, '00:15:00':Interval, 'UTC':Varchar) as $expr1], output_watermarks: [$expr1] }
@@ -149,8 +148,8 @@
   stream_dist_plan: |+
     Fragment 0
     StreamMaterialize { columns: [ts, t1._row_id(hidden)], stream_key: [t1._row_id], pk_columns: [t1._row_id], pk_conflict: NoCheck } { tables: [ Materialize: 4294967294 ] }
-    └── StreamDynamicFilter [append_only] { predicate: (t1.ts < $expr1), output: [t1.ts, t1._row_id], condition_always_relax: true }
-        ├── tables: [ DynamicFilterLeftNotSatisfy: 0, DynamicFilterRight: 1 ]
+    └── StreamDynamicFilter [append_only] { predicate: (t1.ts < $expr1), output: [t1.ts, t1._row_id], cleaned_by_watermark: true }
+        ├── tables: [ DynamicFilterLeft: 0, DynamicFilterRight: 1 ]
         ├── StreamTableScan { table: t1, columns: [t1.ts, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
         │   ├── tables: [ StreamScan: 2 ]
         │   ├── Upstream
@@ -184,7 +183,7 @@
   stream_plan: |-
     StreamMaterialize { columns: [ts, t1._row_id(hidden)], stream_key: [t1._row_id], pk_columns: [t1._row_id], pk_conflict: NoCheck, watermark_columns: [ts] }
     └─StreamDynamicFilter { predicate: (t1.ts >= $expr2), output_watermarks: [t1.ts], output: [t1.ts, t1._row_id], cleaned_by_watermark: true }
-      ├─StreamDynamicFilter { predicate: (t1.ts < $expr1), output: [t1.ts, t1._row_id], condition_always_relax: true }
+      ├─StreamDynamicFilter { predicate: (t1.ts < $expr1), output: [t1.ts, t1._row_id], cleaned_by_watermark: true }
       │ ├─StreamTableScan { table: t1, columns: [t1.ts, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
       │ └─StreamExchange { dist: Broadcast }
       │   └─StreamProject { exprs: [SubtractWithTimeZone(now, '01:00:00':Interval, 'UTC':Varchar) as $expr1], output_watermarks: [$expr1] }
@@ -198,8 +197,7 @@
     ├── tables: [ Materialize: 4294967294 ]
     └── StreamDynamicFilter { predicate: (t1.ts >= $expr2), output_watermarks: [t1.ts], output: [t1.ts, t1._row_id], cleaned_by_watermark: true }
         ├── tables: [ DynamicFilterLeft: 0, DynamicFilterRight: 1 ]
-        ├── StreamDynamicFilter { predicate: (t1.ts < $expr1), output: [t1.ts, t1._row_id], condition_always_relax: true }
-        │   ├── tables: [ DynamicFilterLeftNotSatisfy: 2, DynamicFilterRight: 3 ]
+        ├── StreamDynamicFilter { predicate: (t1.ts < $expr1), output: [t1.ts, t1._row_id], cleaned_by_watermark: true } { tables: [ DynamicFilterLeft: 2, DynamicFilterRight: 3 ] }
         │   ├── StreamTableScan { table: t1, columns: [t1.ts, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
         │   │   ├── tables: [ StreamScan: 4 ]
         │   │   ├── Upstream
@@ -248,7 +246,7 @@
       └─StreamHashJoin { type: Inner, predicate: t1.a = t2.b, output: [t1.a, t1.ta, t2.b, t2.tb, t1._row_id, t2._row_id] }
         ├─StreamExchange { dist: HashShard(t1.a) }
         │ └─StreamDynamicFilter { predicate: (t1.ta >= $expr2), output_watermarks: [t1.ta], output: [t1.a, t1.ta, t1._row_id], cleaned_by_watermark: true }
-        │   ├─StreamDynamicFilter { predicate: (t1.ta < $expr1), output: [t1.a, t1.ta, t1._row_id], condition_always_relax: true }
+        │   ├─StreamDynamicFilter { predicate: (t1.ta < $expr1), output: [t1.a, t1.ta, t1._row_id], cleaned_by_watermark: true }
         │   │ ├─StreamTableScan { table: t1, columns: [t1.a, t1.ta, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
         │   │ └─StreamExchange { dist: Broadcast }
         │   │   └─StreamProject { exprs: [SubtractWithTimeZone(now, '01:00:00':Interval, 'UTC':Varchar) as $expr1], output_watermarks: [$expr1] }
@@ -279,7 +277,7 @@
         │ └─StreamTableScan { table: t2, columns: [t2.b, t2.tb, t2._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t2._row_id], pk: [_row_id], dist: UpstreamHashShard(t2._row_id) }
         └─StreamExchange { dist: HashShard(t1.a) }
           └─StreamDynamicFilter { predicate: (t1.ta >= $expr2), output_watermarks: [t1.ta], output: [t1.a, t1.ta, t1._row_id], cleaned_by_watermark: true }
-            ├─StreamDynamicFilter { predicate: (t1.ta < $expr1), output: [t1.a, t1.ta, t1._row_id], condition_always_relax: true }
+            ├─StreamDynamicFilter { predicate: (t1.ta < $expr1), output: [t1.a, t1.ta, t1._row_id], cleaned_by_watermark: true }
             │ ├─StreamTableScan { table: t1, columns: [t1.a, t1.ta, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
             │ └─StreamExchange { dist: Broadcast }
             │   └─StreamProject { exprs: [SubtractWithTimeZone(now, '01:00:00':Interval, 'UTC':Varchar) as $expr1], output_watermarks: [$expr1] }
@@ -308,7 +306,7 @@
         │ └─StreamTableScan { table: t1, columns: [t1.a, t1.ta, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
         └─StreamExchange { dist: HashShard(t2.b) }
           └─StreamDynamicFilter { predicate: (t2.tb >= $expr2), output_watermarks: [t2.tb], output: [t2.b, t2.tb, t2._row_id], cleaned_by_watermark: true }
-            ├─StreamDynamicFilter { predicate: (t2.tb < $expr1), output: [t2.b, t2.tb, t2._row_id], condition_always_relax: true }
+            ├─StreamDynamicFilter { predicate: (t2.tb < $expr1), output: [t2.b, t2.tb, t2._row_id], cleaned_by_watermark: true }
             │ ├─StreamTableScan { table: t2, columns: [t2.b, t2.tb, t2._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t2._row_id], pk: [_row_id], dist: UpstreamHashShard(t2._row_id) }
             │ └─StreamExchange { dist: Broadcast }
             │   └─StreamProject { exprs: [SubtractWithTimeZone(now, '01:00:00':Interval, 'UTC':Varchar) as $expr1], output_watermarks: [$expr1] }
@@ -416,7 +414,7 @@
     └─StreamUnion { all: true }
       ├─StreamExchange { dist: HashShard(t._row_id, $src, 0:Int32) }
       │ └─StreamProject { exprs: [t.t, t.a, $src, t._row_id, 0:Int32] }
-      │   └─StreamDynamicFilter { predicate: (t.t < $expr2), output: [t.t, t.a, t._row_id, $src], condition_always_relax: true }
+      │   └─StreamDynamicFilter { predicate: (t.t < $expr2), output: [t.t, t.a, t._row_id, $src], cleaned_by_watermark: true }
       │     ├─StreamFilter { predicate: Not((t.a > 1:Int32)) }
       │     │ └─StreamShare { id: 13 }
       │     │   └─StreamUnion { all: true }
@@ -460,3 +458,34 @@
                       └─StreamShare { id: 2 }
                         └─StreamFilter { predicate: IsNotNull(t.a) AND (((Not(IsNull(t.t)) AND Not((t.a < 1:Int32))) OR IsNull(t.t)) OR (t.a < 1:Int32)) }
                           └─StreamTableScan { table: t, columns: [t.t, t.a, t._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t._row_id], pk: [_row_id], dist: UpstreamHashShard(t._row_id) }
+- name: Non-trivial now expression
+  sql: |
+    create table t (ts timestamp with time zone, a int);
+    select * from t where ts + interval '1 hour' > date_trunc('day', now());
+  stream_plan: |-
+    StreamMaterialize { columns: [ts, a, t._row_id(hidden)], stream_key: [t._row_id], pk_columns: [t._row_id], pk_conflict: NoCheck }
+    └─StreamProject { exprs: [t.ts, t.a, t._row_id] }
+      └─StreamDynamicFilter { predicate: ($expr1 > $expr2), output_watermarks: [$expr1], output: [t.ts, t.a, $expr1, t._row_id], cleaned_by_watermark: true }
+        ├─StreamProject { exprs: [t.ts, t.a, AddWithTimeZone(t.ts, '01:00:00':Interval, 'UTC':Varchar) as $expr1, t._row_id] }
+        │ └─StreamTableScan { table: t, columns: [t.ts, t.a, t._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t._row_id], pk: [_row_id], dist: UpstreamHashShard(t._row_id) }
+        └─StreamExchange { dist: Broadcast }
+          └─StreamProject { exprs: [DateTrunc('day':Varchar, now, 'UTC':Varchar) as $expr2], output_watermarks: [$expr2] }
+            └─StreamNow { output: [now] }
+- name: Non-trivial now expression 2
+  sql: |
+    create table t (ts timestamp with time zone, a int);
+    select * from t where ts + interval '1 hour' > date_trunc('day', ('2024-07-18 00:00:00+00:00'::timestamptz - ('2024-07-18 00:00:00+00:00'::timestamptz - now())));
+  stream_plan: |-
+    StreamMaterialize { columns: [ts, a, t._row_id(hidden)], stream_key: [t._row_id], pk_columns: [t._row_id], pk_conflict: NoCheck }
+    └─StreamProject { exprs: [t.ts, t.a, t._row_id] }
+      └─StreamDynamicFilter { predicate: ($expr1 > $expr2), output: [t.ts, t.a, $expr1, t._row_id] }
+        ├─StreamProject { exprs: [t.ts, t.a, AddWithTimeZone(t.ts, '01:00:00':Interval, 'UTC':Varchar) as $expr1, t._row_id] }
+        │ └─StreamTableScan { table: t, columns: [t.ts, t.a, t._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t._row_id], pk: [_row_id], dist: UpstreamHashShard(t._row_id) }
+        └─StreamExchange { dist: Broadcast }
+          └─StreamProject { exprs: [DateTrunc('day':Varchar, SubtractWithTimeZone('2024-07-18 00:00:00+00:00':Timestamptz, ('2024-07-18 00:00:00+00:00':Timestamptz - now), 'UTC':Varchar), 'UTC':Varchar) as $expr2] }
+            └─StreamNow { output: [now] }
+- name: Non-monotonic now expression
+  sql: |
+    create table t (ts timestamp with time zone, a int);
+    select * from t where a > extract(hour from now());
+  stream_error: Conditions containing now must be in the form of `input_expr cmp now_expr` or `now_expr cmp input_expr`, where `input_expr` references a column and contains no `now()`, and `now_expr` is a non-decreasing expression contains `now()`.
diff --git a/src/frontend/planner_test/tests/testdata/output/union.yaml b/src/frontend/planner_test/tests/testdata/output/union.yaml
index ffd31dec73da5..57c65de3cd1f2 100644
--- a/src/frontend/planner_test/tests/testdata/output/union.yaml
+++ b/src/frontend/planner_test/tests/testdata/output/union.yaml
@@ -639,3 +639,106 @@
     ├── distribution key: [ 0, 1, 3 ]
     └── read pk prefix len hint: 3
 
+- name: test corresponding union
+  sql: |
+    create table t1 (a int, b numeric, c bigint);
+    create table t2 (a int, b numeric, y bigint);
+    create table t3 (x int, b numeric, c bigint);
+    select * from t1 union corresponding select * from t2 union all corresponding by (b) select * from t3;
+  batch_plan: |-
+    BatchUnion { all: true }
+    ├─BatchExchange { order: [], dist: Single }
+    │ └─BatchProject { exprs: [t1.b] }
+    │   └─BatchHashAgg { group_key: [t1.a, t1.b], aggs: [] }
+    │     └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+    │       └─BatchUnion { all: true }
+    │         ├─BatchExchange { order: [], dist: Single }
+    │         │ └─BatchScan { table: t1, columns: [t1.a, t1.b], distribution: SomeShard }
+    │         └─BatchExchange { order: [], dist: Single }
+    │           └─BatchScan { table: t2, columns: [t2.a, t2.b], distribution: SomeShard }
+    └─BatchExchange { order: [], dist: Single }
+      └─BatchScan { table: t3, columns: [t3.b], distribution: SomeShard }
+  stream_plan: |-
+    StreamMaterialize { columns: [b, t1.a(hidden), t1.b(hidden), null:Serial(hidden), $src(hidden)], stream_key: [t1.a, t1.b, null:Serial, $src], pk_columns: [t1.a, t1.b, null:Serial, $src], pk_conflict: NoCheck }
+    └─StreamUnion { all: true }
+      ├─StreamExchange { dist: HashShard(t1.a, t1.b, null:Serial, 0:Int32) }
+      │ └─StreamProject { exprs: [t1.b, t1.a, t1.b, null:Serial, 0:Int32], noop_update_hint: true }
+      │   └─StreamHashAgg { group_key: [t1.a, t1.b], aggs: [count] }
+      │     └─StreamExchange { dist: HashShard(t1.a, t1.b) }
+      │       └─StreamUnion { all: true }
+      │         ├─StreamExchange { dist: HashShard(t1._row_id, 0:Int32) }
+      │         │ └─StreamProject { exprs: [t1.a, t1.b, t1._row_id, 0:Int32] }
+      │         │   └─StreamTableScan { table: t1, columns: [t1.a, t1.b, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
+      │         └─StreamExchange { dist: HashShard(t2._row_id, 1:Int32) }
+      │           └─StreamProject { exprs: [t2.a, t2.b, t2._row_id, 1:Int32] }
+      │             └─StreamTableScan { table: t2, columns: [t2.a, t2.b, t2._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t2._row_id], pk: [_row_id], dist: UpstreamHashShard(t2._row_id) }
+      └─StreamExchange { dist: HashShard(null:Int32, null:Decimal, t3._row_id, 1:Int32) }
+        └─StreamProject { exprs: [t3.b, null:Int32, null:Decimal, t3._row_id, 1:Int32] }
+          └─StreamTableScan { table: t3, columns: [t3.b, t3._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t3._row_id], pk: [_row_id], dist: UpstreamHashShard(t3._row_id) }
+  stream_dist_plan: |+
+    Fragment 0
+    StreamMaterialize { columns: [b, t1.a(hidden), t1.b(hidden), null:Serial(hidden), $src(hidden)], stream_key: [t1.a, t1.b, null:Serial, $src], pk_columns: [t1.a, t1.b, null:Serial, $src], pk_conflict: NoCheck }
+    ├── tables: [ Materialize: 4294967294 ]
+    └── StreamUnion { all: true }
+        ├── StreamExchange Hash([1, 2, 3, 4]) from 1
+        └── StreamExchange Hash([1, 2, 3, 4]) from 5
+
+    Fragment 1
+    StreamProject { exprs: [t1.b, t1.a, t1.b, null:Serial, 0:Int32], noop_update_hint: true }
+    └── StreamHashAgg { group_key: [t1.a, t1.b], aggs: [count] } { tables: [ HashAggState: 0 ] }
+        └── StreamExchange Hash([0, 1]) from 2
+
+    Fragment 2
+    StreamUnion { all: true }
+    ├── StreamExchange Hash([2, 3]) from 3
+    └── StreamExchange Hash([2, 3]) from 4
+
+    Fragment 3
+    StreamProject { exprs: [t1.a, t1.b, t1._row_id, 0:Int32] }
+    └── StreamTableScan { table: t1, columns: [t1.a, t1.b, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
+        ├── tables: [ StreamScan: 1 ]
+        ├── Upstream
+        └── BatchPlanNode
+
+    Fragment 4
+    StreamProject { exprs: [t2.a, t2.b, t2._row_id, 1:Int32] }
+    └── StreamTableScan { table: t2, columns: [t2.a, t2.b, t2._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t2._row_id], pk: [_row_id], dist: UpstreamHashShard(t2._row_id) }
+        ├── tables: [ StreamScan: 2 ]
+        ├── Upstream
+        └── BatchPlanNode
+
+    Fragment 5
+    StreamProject { exprs: [t3.b, null:Int32, null:Decimal, t3._row_id, 1:Int32] }
+    └── StreamTableScan { table: t3, columns: [t3.b, t3._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t3._row_id], pk: [_row_id], dist: UpstreamHashShard(t3._row_id) }
+        ├── tables: [ StreamScan: 3 ]
+        ├── Upstream
+        └── BatchPlanNode
+
+    Table 0 { columns: [ t1_a, t1_b, count ], primary key: [ $0 ASC, $1 ASC ], value indices: [ 2 ], distribution key: [ 0, 1 ], read pk prefix len hint: 2 }
+
+    Table 1 { columns: [ vnode, _row_id, backfill_finished, row_count ], primary key: [ $0 ASC ], value indices: [ 1, 2, 3 ], distribution key: [ 0 ], read pk prefix len hint: 1, vnode column idx: 0 }
+
+    Table 2 { columns: [ vnode, _row_id, backfill_finished, row_count ], primary key: [ $0 ASC ], value indices: [ 1, 2, 3 ], distribution key: [ 0 ], read pk prefix len hint: 1, vnode column idx: 0 }
+
+    Table 3 { columns: [ vnode, _row_id, backfill_finished, row_count ], primary key: [ $0 ASC ], value indices: [ 1, 2, 3 ], distribution key: [ 0 ], read pk prefix len hint: 1, vnode column idx: 0 }
+
+    Table 4294967294
+    ├── columns: [ b, t1.a, t1.b, null:Serial, $src ]
+    ├── primary key: [ $1 ASC, $2 ASC, $3 ASC, $4 ASC ]
+    ├── value indices: [ 0, 1, 2, 3, 4 ]
+    ├── distribution key: [ 1, 2, 3, 4 ]
+    └── read pk prefix len hint: 4
+
+- name: test corresponding union error - corresponding list
+  sql: |
+    create table t1 (a int, b numeric, c bigint);
+    create table t2 (a int, b numeric, y bigint);
+    create table t3 (x int, b numeric, c bigint);
+    select * from t1 union corresponding select * from t2 union all corresponding by (c) select * from t3;
+  binder_error: 'Invalid input syntax: Column name `c` in CORRESPONDING BY is not found in a side of the UNION operation. It shall be included in both sides.'
+- name: test corresponding union error - duplicate names
+  sql: |
+    create table t1 (a int, b numeric, c bigint);
+    create table t2 (a int, b numeric, y bigint);
+    select a, b as a from t1 union corresponding select * from t2;
+  binder_error: 'Invalid input syntax: Duplicated column name `a` in a column list of the query in a UNION operation. Column list of the query: ("a", "a").'
diff --git a/src/frontend/src/binder/create_view.rs b/src/frontend/src/binder/create_view.rs
new file mode 100644
index 0000000000000..92c0a93edd4c7
--- /dev/null
+++ b/src/frontend/src/binder/create_view.rs
@@ -0,0 +1,62 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use risingwave_sqlparser::ast::{EmitMode, Ident, ObjectName, SqlOption};
+
+use crate::binder::statement::RewriteExprsRecursive;
+use crate::binder::BoundQuery;
+use crate::expr::ExprRewriter;
+
+/// Represents a bounded `CREATE MATERIALIZED VIEW` statement.
+#[derive(Debug, Clone)]
+pub struct BoundCreateView {
+    pub or_replace: bool,
+    pub materialized: bool, // always true currently
+    pub if_not_exists: bool,
+    pub name: ObjectName,
+    pub columns: Vec<Ident>,
+    pub query: Box<BoundQuery>, // reuse the BoundQuery struct
+    pub emit_mode: Option<EmitMode>,
+    pub with_options: Vec<SqlOption>,
+}
+
+impl BoundCreateView {
+    pub fn new(
+        or_replace: bool,
+        materialized: bool,
+        if_not_exists: bool,
+        name: ObjectName,
+        columns: Vec<Ident>,
+        query: BoundQuery,
+        emit_mode: Option<EmitMode>,
+        with_options: Vec<SqlOption>,
+    ) -> Self {
+        Self {
+            or_replace,
+            materialized,
+            if_not_exists,
+            name,
+            columns,
+            query: Box::new(query),
+            emit_mode,
+            with_options,
+        }
+    }
+}
+
+impl RewriteExprsRecursive for BoundCreateView {
+    fn rewrite_exprs_recursive(&mut self, rewriter: &mut impl ExprRewriter) {
+        self.query.rewrite_exprs_recursive(rewriter);
+    }
+}
diff --git a/src/frontend/src/binder/expr/function.rs b/src/frontend/src/binder/expr/function.rs
deleted file mode 100644
index 95466021863e4..0000000000000
--- a/src/frontend/src/binder/expr/function.rs
+++ /dev/null
@@ -1,1677 +0,0 @@
-// Copyright 2024 RisingWave Labs
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::collections::HashMap;
-use std::iter::once;
-use std::str::FromStr;
-use std::sync::{Arc, LazyLock};
-
-use bk_tree::{metrics, BKTree};
-use itertools::Itertools;
-use risingwave_common::array::ListValue;
-use risingwave_common::catalog::{INFORMATION_SCHEMA_SCHEMA_NAME, PG_CATALOG_SCHEMA_NAME};
-use risingwave_common::session_config::USER_NAME_WILD_CARD;
-use risingwave_common::types::{data_types, DataType, ScalarImpl, Timestamptz};
-use risingwave_common::{bail_not_implemented, current_cluster_version, must_match, no_function};
-use risingwave_expr::aggregate::{agg_kinds, AggKind};
-use risingwave_expr::window_function::{
-    Frame, FrameBound, FrameBounds, FrameExclusion, RangeFrameBounds, RangeFrameOffset,
-    RowsFrameBounds, SessionFrameBounds, SessionFrameGap, WindowFuncKind,
-};
-use risingwave_sqlparser::ast::{
-    self, Function, FunctionArg, FunctionArgExpr, Ident, WindowFrameBound, WindowFrameBounds,
-    WindowFrameExclusion, WindowFrameUnits, WindowSpec,
-};
-use risingwave_sqlparser::parser::ParserError;
-use thiserror_ext::AsReport;
-
-use crate::binder::bind_context::Clause;
-use crate::binder::{Binder, UdfContext};
-use crate::catalog::function_catalog::FunctionCatalog;
-use crate::error::{ErrorCode, Result, RwError};
-use crate::expr::{
-    AggCall, CastContext, Expr, ExprImpl, ExprType, FunctionCall, FunctionCallWithLambda, Literal,
-    Now, OrderBy, TableFunction, TableFunctionType, UserDefinedFunction, WindowFunction,
-};
-use crate::utils::Condition;
-
-// Defines system functions that without args, ref: https://www.postgresql.org/docs/current/functions-info.html
-pub const SYS_FUNCTION_WITHOUT_ARGS: &[&str] = &[
-    "session_user",
-    "user",
-    "current_user",
-    "current_role",
-    "current_schema",
-    "current_timestamp",
-];
-
-/// The global max calling depth for the global counter in `udf_context`
-/// To reduce the chance that the current running rw thread
-/// be killed by os, the current allowance depth of calling
-/// stack is set to `16`.
-const SQL_UDF_MAX_CALLING_DEPTH: u32 = 16;
-
-impl Binder {
-    pub(in crate::binder) fn bind_function(&mut self, f: Function) -> Result<ExprImpl> {
-        let function_name = match f.name.0.as_slice() {
-            [name] => name.real_value(),
-            [schema, name] => {
-                let schema_name = schema.real_value();
-                if schema_name == PG_CATALOG_SCHEMA_NAME {
-                    // pg_catalog is always effectively part of the search path, so we can always bind the function.
-                    // Ref: https://www.postgresql.org/docs/current/ddl-schemas.html#DDL-SCHEMAS-CATALOG
-                    name.real_value()
-                } else if schema_name == INFORMATION_SCHEMA_SCHEMA_NAME {
-                    // definition of information_schema: https://github.com/postgres/postgres/blob/e0b2eed047df9045664da6f724cb42c10f8b12f0/src/backend/catalog/information_schema.sql
-                    //
-                    // FIXME: handle schema correctly, so that the functions are hidden if the schema is not in the search path.
-                    let function_name = name.real_value();
-                    if function_name != "_pg_expandarray" {
-                        bail_not_implemented!(
-                            issue = 12422,
-                            "Unsupported function name under schema: {}",
-                            schema_name
-                        );
-                    }
-                    function_name
-                } else {
-                    bail_not_implemented!(
-                        issue = 12422,
-                        "Unsupported function name under schema: {}",
-                        schema_name
-                    );
-                }
-            }
-            _ => bail_not_implemented!(issue = 112, "qualified function {}", f.name),
-        };
-
-        // FIXME: This is a hack to support [Bytebase queries](https://github.com/TennyZhuang/bytebase/blob/4a26f7c62b80e86e58ad2f77063138dc2f420623/backend/plugin/db/pg/sync.go#L549).
-        // Bytebase widely used the pattern like `obj_description(format('%s.%s',
-        // quote_ident(idx.schemaname), quote_ident(idx.indexname))::regclass) AS comment` to
-        // retrieve object comment, however we don't support casting a non-literal expression to
-        // regclass. We just hack the `obj_description` and `col_description` here, to disable it to
-        // bind its arguments.
-        if function_name == "obj_description" || function_name == "col_description" {
-            return Ok(ExprImpl::literal_varchar("".to_string()));
-        }
-        if function_name == "array_transform" {
-            // For type inference, we need to bind the array type first.
-            return self.bind_array_transform(f);
-        }
-
-        let mut inputs: Vec<_> = f
-            .args
-            .iter()
-            .map(|arg| self.bind_function_arg(arg.clone()))
-            .flatten_ok()
-            .try_collect()?;
-
-        // user defined function
-        // TODO: resolve schema name https://github.com/risingwavelabs/risingwave/issues/12422
-        if let Ok(schema) = self.first_valid_schema()
-            && let Some(func) = schema.get_function_by_name_inputs(&function_name, &mut inputs)
-        {
-            use crate::catalog::function_catalog::FunctionKind::*;
-
-            if func.language == "sql" {
-                if func.body.is_none() {
-                    return Err(ErrorCode::InvalidInputSyntax(
-                        "`body` must exist for sql udf".to_string(),
-                    )
-                    .into());
-                }
-
-                // This represents the current user defined function is `language sql`
-                let parse_result = risingwave_sqlparser::parser::Parser::parse_sql(
-                    func.body.as_ref().unwrap().as_str(),
-                );
-                if let Err(ParserError::ParserError(err)) | Err(ParserError::TokenizerError(err)) =
-                    parse_result
-                {
-                    // Here we just return the original parse error message
-                    return Err(ErrorCode::InvalidInputSyntax(err).into());
-                }
-
-                debug_assert!(parse_result.is_ok());
-
-                // We can safely unwrap here
-                let ast = parse_result.unwrap();
-
-                // Stash the current `udf_context`
-                // Note that the `udf_context` may be empty,
-                // if the current binding is the root (top-most) sql udf.
-                // In this case the empty context will be stashed
-                // and restored later, no need to maintain other flags.
-                let stashed_udf_context = self.udf_context.get_context();
-
-                // The actual inline logic for sql udf
-                // Note that we will always create new udf context for each sql udf
-                let Ok(context) = UdfContext::create_udf_context(&f.args, &Arc::clone(func)) else {
-                    return Err(ErrorCode::InvalidInputSyntax(
-                        "failed to create the `udf_context`, please recheck your function definition and syntax".to_string()
-                    )
-                    .into());
-                };
-
-                let mut udf_context = HashMap::new();
-                for (c, e) in context {
-                    // Note that we need to bind the args before actual delve in the function body
-                    // This will update the context in the subsequent inner calling function
-                    // e.g.,
-                    // - create function print(INT) returns int language sql as 'select $1';
-                    // - create function print_add_one(INT) returns int language sql as 'select print($1 + 1)';
-                    // - select print_add_one(1); # The result should be 2 instead of 1.
-                    // Without the pre-binding here, the ($1 + 1) will not be correctly populated,
-                    // causing the result to always be 1.
-                    let Ok(e) = self.bind_expr(e) else {
-                        return Err(ErrorCode::BindError(
-                            "failed to bind the argument, please recheck the syntax".to_string(),
-                        )
-                        .into());
-                    };
-                    udf_context.insert(c, e);
-                }
-                self.udf_context.update_context(udf_context);
-
-                // Check for potential recursive calling
-                if self.udf_context.global_count() >= SQL_UDF_MAX_CALLING_DEPTH {
-                    return Err(ErrorCode::BindError(format!(
-                        "function {} calling stack depth limit exceeded",
-                        &function_name
-                    ))
-                    .into());
-                } else {
-                    // Update the status for the global counter
-                    self.udf_context.incr_global_count();
-                }
-
-                if let Ok(expr) = UdfContext::extract_udf_expression(ast) {
-                    let bind_result = self.bind_expr(expr);
-
-                    // We should properly decrement global count after a successful binding
-                    // Since the subsequent probe operation in `bind_column` or
-                    // `bind_parameter` relies on global counting
-                    self.udf_context.decr_global_count();
-
-                    // Restore context information for subsequent binding
-                    self.udf_context.update_context(stashed_udf_context);
-
-                    return bind_result;
-                } else {
-                    return Err(ErrorCode::InvalidInputSyntax(
-                        "failed to parse the input query and extract the udf expression,
-                        please recheck the syntax"
-                            .to_string(),
-                    )
-                    .into());
-                }
-            } else {
-                match &func.kind {
-                    Scalar { .. } => {
-                        return Ok(UserDefinedFunction::new(func.clone(), inputs).into())
-                    }
-                    Table { .. } => {
-                        self.ensure_table_function_allowed()?;
-                        return Ok(TableFunction::new_user_defined(func.clone(), inputs).into());
-                    }
-                    Aggregate => {
-                        return self.bind_agg(f, AggKind::UserDefined, Some(func.clone()));
-                    }
-                }
-            }
-        }
-
-        // agg calls
-        if f.over.is_none()
-            && let Ok(kind) = function_name.parse()
-        {
-            return self.bind_agg(f, kind, None);
-        }
-
-        if f.distinct || !f.order_by.is_empty() || f.filter.is_some() {
-            return Err(ErrorCode::InvalidInputSyntax(format!(
-                    "DISTINCT, ORDER BY or FILTER is only allowed in aggregation functions, but `{}` is not an aggregation function", function_name
-                )
-                )
-                .into());
-        }
-
-        // window function
-        let window_func_kind = WindowFuncKind::from_str(function_name.as_str());
-        if let Ok(kind) = window_func_kind {
-            if let Some(window_spec) = f.over {
-                return self.bind_window_function(kind, inputs, window_spec);
-            }
-            return Err(ErrorCode::InvalidInputSyntax(format!(
-                "Window function `{}` must have OVER clause",
-                function_name
-            ))
-            .into());
-        } else if f.over.is_some() {
-            bail_not_implemented!(
-                issue = 8961,
-                "Unrecognized window function: {}",
-                function_name
-            );
-        }
-
-        // table function
-        if let Ok(function_type) = TableFunctionType::from_str(function_name.as_str()) {
-            self.ensure_table_function_allowed()?;
-            return Ok(TableFunction::new(function_type, inputs)?.into());
-        }
-
-        self.bind_builtin_scalar_function(function_name.as_str(), inputs, f.variadic)
-    }
-
-    fn bind_array_transform(&mut self, f: Function) -> Result<ExprImpl> {
-        let [array, lambda] = <[FunctionArg; 2]>::try_from(f.args).map_err(|args| -> RwError {
-            ErrorCode::BindError(format!(
-                "`array_transform` expect two inputs `array` and `lambda`, but {} were given",
-                args.len()
-            ))
-            .into()
-        })?;
-
-        let bound_array = self.bind_function_arg(array)?;
-        let [bound_array] = <[ExprImpl; 1]>::try_from(bound_array).map_err(|bound_array| -> RwError {
-            ErrorCode::BindError(format!("The `array` argument for `array_transform` should be bound to one argument, but {} were got", bound_array.len()))
-                .into()
-        })?;
-
-        let inner_ty = match bound_array.return_type() {
-            DataType::List(ty) => *ty,
-            real_type => {
-                return Err(ErrorCode::BindError(format!(
-                "The `array` argument for `array_transform` should be an array, but {} were got",
-                real_type
-            ))
-                .into())
-            }
-        };
-
-        let ast::FunctionArgExpr::Expr(ast::Expr::LambdaFunction {
-            args: lambda_args,
-            body: lambda_body,
-        }) = lambda.get_expr()
-        else {
-            return Err(ErrorCode::BindError(
-                "The `lambda` argument for `array_transform` should be a lambda function"
-                    .to_string(),
-            )
-            .into());
-        };
-
-        let [lambda_arg] = <[Ident; 1]>::try_from(lambda_args).map_err(|args| -> RwError {
-            ErrorCode::BindError(format!(
-                "The `lambda` argument for `array_transform` should be a lambda function with one argument, but {} were given",
-                args.len()
-            ))
-            .into()
-        })?;
-
-        let bound_lambda = self.bind_unary_lambda_function(inner_ty, lambda_arg, *lambda_body)?;
-
-        let lambda_ret_type = bound_lambda.return_type();
-        let transform_ret_type = DataType::List(Box::new(lambda_ret_type));
-
-        Ok(ExprImpl::FunctionCallWithLambda(Box::new(
-            FunctionCallWithLambda::new_unchecked(
-                ExprType::ArrayTransform,
-                vec![bound_array],
-                bound_lambda,
-                transform_ret_type,
-            ),
-        )))
-    }
-
-    fn bind_unary_lambda_function(
-        &mut self,
-        input_ty: DataType,
-        arg: Ident,
-        body: ast::Expr,
-    ) -> Result<ExprImpl> {
-        let lambda_args = HashMap::from([(arg.real_value(), (0usize, input_ty))]);
-        let orig_lambda_args = self.context.lambda_args.replace(lambda_args);
-        let body = self.bind_expr_inner(body)?;
-        self.context.lambda_args = orig_lambda_args;
-
-        Ok(body)
-    }
-
-    pub(super) fn bind_agg(
-        &mut self,
-        f: Function,
-        kind: AggKind,
-        user_defined: Option<Arc<FunctionCatalog>>,
-    ) -> Result<ExprImpl> {
-        self.ensure_aggregate_allowed()?;
-
-        let distinct = f.distinct;
-        let filter_expr = f.filter.clone();
-
-        let (direct_args, args, order_by) = if matches!(kind, agg_kinds::ordered_set!()) {
-            self.bind_ordered_set_agg(f, kind)?
-        } else {
-            self.bind_normal_agg(f, kind)?
-        };
-
-        let filter = match filter_expr {
-            Some(filter) => {
-                let mut clause = Some(Clause::Filter);
-                std::mem::swap(&mut self.context.clause, &mut clause);
-                let expr = self
-                    .bind_expr_inner(*filter)
-                    .and_then(|expr| expr.enforce_bool_clause("FILTER"))?;
-                self.context.clause = clause;
-                if expr.has_subquery() {
-                    bail_not_implemented!("subquery in filter clause");
-                }
-                if expr.has_agg_call() {
-                    bail_not_implemented!("aggregation function in filter clause");
-                }
-                if expr.has_table_function() {
-                    bail_not_implemented!("table function in filter clause");
-                }
-                Condition::with_expr(expr)
-            }
-            None => Condition::true_cond(),
-        };
-
-        if let Some(user_defined) = user_defined {
-            Ok(AggCall::new_user_defined(
-                args,
-                distinct,
-                order_by,
-                filter,
-                direct_args,
-                user_defined,
-            )?
-            .into())
-        } else {
-            Ok(ExprImpl::AggCall(Box::new(AggCall::new(
-                kind,
-                args,
-                distinct,
-                order_by,
-                filter,
-                direct_args,
-            )?)))
-        }
-    }
-
-    fn bind_ordered_set_agg(
-        &mut self,
-        f: Function,
-        kind: AggKind,
-    ) -> Result<(Vec<Literal>, Vec<ExprImpl>, OrderBy)> {
-        // Syntax:
-        // aggregate_name ( [ expression [ , ... ] ] ) WITHIN GROUP ( order_by_clause ) [ FILTER
-        // ( WHERE filter_clause ) ]
-
-        assert!(matches!(kind, agg_kinds::ordered_set!()));
-
-        if !f.order_by.is_empty() {
-            return Err(ErrorCode::InvalidInputSyntax(format!(
-                "ORDER BY is not allowed for ordered-set aggregation `{}`",
-                kind
-            ))
-            .into());
-        }
-        if f.distinct {
-            return Err(ErrorCode::InvalidInputSyntax(format!(
-                "DISTINCT is not allowed for ordered-set aggregation `{}`",
-                kind
-            ))
-            .into());
-        }
-
-        let within_group = *f.within_group.ok_or_else(|| {
-            ErrorCode::InvalidInputSyntax(format!(
-                "WITHIN GROUP is expected for ordered-set aggregation `{}`",
-                kind
-            ))
-        })?;
-
-        let mut direct_args: Vec<_> = f
-            .args
-            .into_iter()
-            .map(|arg| self.bind_function_arg(arg))
-            .flatten_ok()
-            .try_collect()?;
-        let mut args =
-            self.bind_function_expr_arg(FunctionArgExpr::Expr(within_group.expr.clone()))?;
-        let order_by = OrderBy::new(vec![self.bind_order_by_expr(within_group)?]);
-
-        // check signature and do implicit cast
-        match (kind, direct_args.as_mut_slice(), args.as_mut_slice()) {
-            (AggKind::PercentileCont | AggKind::PercentileDisc, [fraction], [arg]) => {
-                if fraction.cast_implicit_mut(DataType::Float64).is_err() {
-                    return Err(ErrorCode::InvalidInputSyntax(format!(
-                        "direct arg in `{}` must be castable to float64",
-                        kind
-                    ))
-                    .into());
-                }
-
-                let Some(Ok(fraction_datum)) = fraction.try_fold_const() else {
-                    bail_not_implemented!(
-                        issue = 14079,
-                        "variable as direct argument of ordered-set aggregate",
-                    );
-                };
-
-                if let Some(ref fraction_value) = fraction_datum
-                    && !(0.0..=1.0).contains(&fraction_value.as_float64().0)
-                {
-                    return Err(ErrorCode::InvalidInputSyntax(format!(
-                        "direct arg in `{}` must between 0.0 and 1.0",
-                        kind
-                    ))
-                    .into());
-                }
-                // note that the fraction can be NULL
-                *fraction = Literal::new(fraction_datum, DataType::Float64).into();
-
-                if kind == AggKind::PercentileCont {
-                    arg.cast_implicit_mut(DataType::Float64).map_err(|_| {
-                        ErrorCode::InvalidInputSyntax(format!(
-                            "arg in `{}` must be castable to float64",
-                            kind
-                        ))
-                    })?;
-                }
-            }
-            (AggKind::Mode, [], [_arg]) => {}
-            _ => {
-                return Err(ErrorCode::InvalidInputSyntax(format!(
-                    "invalid direct args or within group argument for `{}` aggregation",
-                    kind
-                ))
-                .into())
-            }
-        }
-
-        Ok((
-            direct_args
-                .into_iter()
-                .map(|arg| *arg.into_literal().unwrap())
-                .collect(),
-            args,
-            order_by,
-        ))
-    }
-
-    fn bind_normal_agg(
-        &mut self,
-        f: Function,
-        kind: AggKind,
-    ) -> Result<(Vec<Literal>, Vec<ExprImpl>, OrderBy)> {
-        // Syntax:
-        // aggregate_name (expression [ , ... ] [ order_by_clause ] ) [ FILTER ( WHERE
-        //   filter_clause ) ]
-        // aggregate_name (ALL expression [ , ... ] [ order_by_clause ] ) [ FILTER ( WHERE
-        //   filter_clause ) ]
-        // aggregate_name (DISTINCT expression [ , ... ] [ order_by_clause ] ) [ FILTER ( WHERE
-        //   filter_clause ) ]
-        // aggregate_name ( * ) [ FILTER ( WHERE filter_clause ) ]
-
-        assert!(!matches!(kind, agg_kinds::ordered_set!()));
-
-        if f.within_group.is_some() {
-            return Err(ErrorCode::InvalidInputSyntax(format!(
-                "WITHIN GROUP is not allowed for non-ordered-set aggregation `{}`",
-                kind
-            ))
-            .into());
-        }
-
-        let args: Vec<_> = f
-            .args
-            .iter()
-            .map(|arg| self.bind_function_arg(arg.clone()))
-            .flatten_ok()
-            .try_collect()?;
-        let order_by = OrderBy::new(
-            f.order_by
-                .into_iter()
-                .map(|e| self.bind_order_by_expr(e))
-                .try_collect()?,
-        );
-
-        if f.distinct {
-            if kind == AggKind::ApproxCountDistinct {
-                return Err(ErrorCode::InvalidInputSyntax(format!(
-                    "DISTINCT is not allowed for approximate aggregation `{}`",
-                    kind
-                ))
-                .into());
-            }
-
-            if args.is_empty() {
-                return Err(ErrorCode::InvalidInputSyntax(format!(
-                    "DISTINCT is not allowed for aggregate function `{}` without args",
-                    kind
-                ))
-                .into());
-            }
-
-            // restrict arguments[1..] to be constant because we don't support multiple distinct key
-            // indices for now
-            if args.iter().skip(1).any(|arg| arg.as_literal().is_none()) {
-                bail_not_implemented!("non-constant arguments other than the first one for DISTINCT aggregation is not supported now");
-            }
-
-            // restrict ORDER BY to align with PG, which says:
-            // > If DISTINCT is specified in addition to an order_by_clause, then all the ORDER BY
-            // > expressions must match regular arguments of the aggregate; that is, you cannot sort
-            // > on an expression that is not included in the DISTINCT list.
-            if !order_by.sort_exprs.iter().all(|e| args.contains(&e.expr)) {
-                return Err(ErrorCode::InvalidInputSyntax(format!(
-                    "ORDER BY expressions must match regular arguments of the aggregate for `{}` when DISTINCT is provided",
-                    kind
-                ))
-                .into());
-            }
-        }
-
-        Ok((vec![], args, order_by))
-    }
-
-    /// Bind window function calls according to PostgreSQL syntax.
-    /// See <https://www.postgresql.org/docs/current/sql-expressions.html#SYNTAX-WINDOW-FUNCTIONS> for syntax detail.
-    pub(super) fn bind_window_function(
-        &mut self,
-        kind: WindowFuncKind,
-        inputs: Vec<ExprImpl>,
-        WindowSpec {
-            partition_by,
-            order_by,
-            window_frame,
-        }: WindowSpec,
-    ) -> Result<ExprImpl> {
-        self.ensure_window_function_allowed()?;
-        let partition_by = partition_by
-            .into_iter()
-            .map(|arg| self.bind_expr_inner(arg))
-            .try_collect()?;
-        let order_by = OrderBy::new(
-            order_by
-                .into_iter()
-                .map(|order_by_expr| self.bind_order_by_expr(order_by_expr))
-                .collect::<Result<_>>()?,
-        );
-        let frame = if let Some(frame) = window_frame {
-            let exclusion = if let Some(exclusion) = frame.exclusion {
-                match exclusion {
-                    WindowFrameExclusion::CurrentRow => FrameExclusion::CurrentRow,
-                    WindowFrameExclusion::Group | WindowFrameExclusion::Ties => {
-                        bail_not_implemented!(
-                            issue = 9124,
-                            "window frame exclusion `{}` is not supported yet",
-                            exclusion
-                        );
-                    }
-                    WindowFrameExclusion::NoOthers => FrameExclusion::NoOthers,
-                }
-            } else {
-                FrameExclusion::NoOthers
-            };
-            let bounds = match frame.units {
-                WindowFrameUnits::Rows => {
-                    let (start, end) = must_match!(frame.bounds, WindowFrameBounds::Bounds { start, end } => (start, end));
-                    let (start, end) = self.bind_window_frame_usize_bounds(start, end)?;
-                    FrameBounds::Rows(RowsFrameBounds { start, end })
-                }
-                unit @ (WindowFrameUnits::Range | WindowFrameUnits::Session) => {
-                    let order_by_expr = order_by
-                        .sort_exprs
-                        .iter()
-                        // for `RANGE | SESSION` frame, there should be exactly one `ORDER BY` column
-                        .exactly_one()
-                        .map_err(|_| {
-                            ErrorCode::InvalidInputSyntax(format!(
-                                "there should be exactly one ordering column for `{}` frame",
-                                unit
-                            ))
-                        })?;
-                    let order_data_type = order_by_expr.expr.return_type();
-                    let order_type = order_by_expr.order_type;
-
-                    let offset_data_type = match &order_data_type {
-                        // for numeric ordering columns, `offset`/`gap` should be the same type
-                        // NOTE: actually in PG it can be a larger type, but we don't support this here
-                        t @ data_types::range_frame_numeric!() => t.clone(),
-                        // for datetime ordering columns, `offset`/`gap` should be interval
-                        t @ data_types::range_frame_datetime!() => {
-                            if matches!(t, DataType::Date | DataType::Time) {
-                                bail_not_implemented!(
-                                    "`{}` frame with offset of type `{}` is not implemented yet, please manually cast the `ORDER BY` column to `timestamp`",
-                                    unit,
-                                    t
-                                );
-                            }
-                            DataType::Interval
-                        }
-                        // other types are not supported
-                        t => {
-                            return Err(ErrorCode::NotSupported(
-                                format!(
-                                    "`{}` frame with offset of type `{}` is not supported",
-                                    unit, t
-                                ),
-                                "Please re-consider the `ORDER BY` column".to_string(),
-                            )
-                            .into())
-                        }
-                    };
-
-                    if unit == WindowFrameUnits::Range {
-                        let (start, end) = must_match!(frame.bounds, WindowFrameBounds::Bounds { start, end } => (start, end));
-                        let (start, end) = self.bind_window_frame_scalar_impl_bounds(
-                            start,
-                            end,
-                            &offset_data_type,
-                        )?;
-                        FrameBounds::Range(RangeFrameBounds {
-                            order_data_type,
-                            order_type,
-                            offset_data_type,
-                            start: start.map(RangeFrameOffset::new),
-                            end: end.map(RangeFrameOffset::new),
-                        })
-                    } else {
-                        let gap = must_match!(frame.bounds, WindowFrameBounds::Gap(gap) => gap);
-                        let gap_value =
-                            self.bind_window_frame_bound_offset(*gap, offset_data_type.clone())?;
-                        FrameBounds::Session(SessionFrameBounds {
-                            order_data_type,
-                            order_type,
-                            gap_data_type: offset_data_type,
-                            gap: SessionFrameGap::new(gap_value),
-                        })
-                    }
-                }
-                WindowFrameUnits::Groups => {
-                    bail_not_implemented!(
-                        issue = 9124,
-                        "window frame in `GROUPS` mode is not supported yet",
-                    );
-                }
-            };
-
-            // Validate the frame bounds, may return `ExprError` to user if the bounds given are not valid.
-            bounds.validate()?;
-
-            Some(Frame { bounds, exclusion })
-        } else {
-            None
-        };
-        Ok(WindowFunction::new(kind, partition_by, order_by, inputs, frame)?.into())
-    }
-
-    fn bind_window_frame_usize_bounds(
-        &mut self,
-        start: WindowFrameBound,
-        end: Option<WindowFrameBound>,
-    ) -> Result<(FrameBound<usize>, FrameBound<usize>)> {
-        let mut convert_offset = |offset: Box<ast::Expr>| -> Result<usize> {
-            let offset = self
-                .bind_window_frame_bound_offset(*offset, DataType::Int64)?
-                .into_int64();
-            if offset < 0 {
-                return Err(ErrorCode::InvalidInputSyntax(
-                    "offset in window frame bounds must be non-negative".to_string(),
-                )
-                .into());
-            }
-            Ok(offset as usize)
-        };
-        let mut convert_bound = |bound| -> Result<FrameBound<usize>> {
-            Ok(match bound {
-                WindowFrameBound::CurrentRow => FrameBound::CurrentRow,
-                WindowFrameBound::Preceding(None) => FrameBound::UnboundedPreceding,
-                WindowFrameBound::Preceding(Some(offset)) => {
-                    FrameBound::Preceding(convert_offset(offset)?)
-                }
-                WindowFrameBound::Following(None) => FrameBound::UnboundedFollowing,
-                WindowFrameBound::Following(Some(offset)) => {
-                    FrameBound::Following(convert_offset(offset)?)
-                }
-            })
-        };
-        let start = convert_bound(start)?;
-        let end = if let Some(end_bound) = end {
-            convert_bound(end_bound)?
-        } else {
-            FrameBound::CurrentRow
-        };
-        Ok((start, end))
-    }
-
-    fn bind_window_frame_scalar_impl_bounds(
-        &mut self,
-        start: WindowFrameBound,
-        end: Option<WindowFrameBound>,
-        offset_data_type: &DataType,
-    ) -> Result<(FrameBound<ScalarImpl>, FrameBound<ScalarImpl>)> {
-        let mut convert_bound = |bound| -> Result<FrameBound<_>> {
-            Ok(match bound {
-                WindowFrameBound::CurrentRow => FrameBound::CurrentRow,
-                WindowFrameBound::Preceding(None) => FrameBound::UnboundedPreceding,
-                WindowFrameBound::Preceding(Some(offset)) => FrameBound::Preceding(
-                    self.bind_window_frame_bound_offset(*offset, offset_data_type.clone())?,
-                ),
-                WindowFrameBound::Following(None) => FrameBound::UnboundedFollowing,
-                WindowFrameBound::Following(Some(offset)) => FrameBound::Following(
-                    self.bind_window_frame_bound_offset(*offset, offset_data_type.clone())?,
-                ),
-            })
-        };
-        let start = convert_bound(start)?;
-        let end = if let Some(end_bound) = end {
-            convert_bound(end_bound)?
-        } else {
-            FrameBound::CurrentRow
-        };
-        Ok((start, end))
-    }
-
-    fn bind_window_frame_bound_offset(
-        &mut self,
-        offset: ast::Expr,
-        cast_to: DataType,
-    ) -> Result<ScalarImpl> {
-        let mut offset = self.bind_expr(offset)?;
-        if !offset.is_const() {
-            return Err(ErrorCode::InvalidInputSyntax(
-                "offset/gap in window frame bounds must be constant".to_string(),
-            )
-            .into());
-        }
-        if offset.cast_implicit_mut(cast_to.clone()).is_err() {
-            return Err(ErrorCode::InvalidInputSyntax(format!(
-                "offset/gap in window frame bounds must be castable to {}",
-                cast_to
-            ))
-            .into());
-        }
-        let offset = offset.fold_const()?;
-        let Some(offset) = offset else {
-            return Err(ErrorCode::InvalidInputSyntax(
-                "offset/gap in window frame bounds must not be NULL".to_string(),
-            )
-            .into());
-        };
-        Ok(offset)
-    }
-
-    fn bind_builtin_scalar_function(
-        &mut self,
-        function_name: &str,
-        inputs: Vec<ExprImpl>,
-        variadic: bool,
-    ) -> Result<ExprImpl> {
-        type Inputs = Vec<ExprImpl>;
-
-        type Handle = Box<dyn Fn(&mut Binder, Inputs) -> Result<ExprImpl> + Sync + Send>;
-
-        fn rewrite(r#type: ExprType, rewriter: fn(Inputs) -> Result<Inputs>) -> Handle {
-            Box::new(move |_binder, mut inputs| {
-                inputs = (rewriter)(inputs)?;
-                Ok(FunctionCall::new(r#type, inputs)?.into())
-            })
-        }
-
-        fn raw_call(r#type: ExprType) -> Handle {
-            rewrite(r#type, Ok)
-        }
-
-        fn guard_by_len(expected_len: usize, handle: Handle) -> Handle {
-            Box::new(move |binder, inputs| {
-                if inputs.len() == expected_len {
-                    handle(binder, inputs)
-                } else {
-                    Err(ErrorCode::ExprError("unexpected arguments number".into()).into())
-                }
-            })
-        }
-
-        fn raw<F: Fn(&mut Binder, Inputs) -> Result<ExprImpl> + Sync + Send + 'static>(
-            f: F,
-        ) -> Handle {
-            Box::new(f)
-        }
-
-        fn dispatch_by_len(mapping: Vec<(usize, Handle)>) -> Handle {
-            Box::new(move |binder, inputs| {
-                for (len, handle) in &mapping {
-                    if inputs.len() == *len {
-                        return handle(binder, inputs);
-                    }
-                }
-                Err(ErrorCode::ExprError("unexpected arguments number".into()).into())
-            })
-        }
-
-        fn raw_literal(literal: ExprImpl) -> Handle {
-            Box::new(move |_binder, _inputs| Ok(literal.clone()))
-        }
-
-        fn now() -> Handle {
-            guard_by_len(
-                0,
-                raw(move |binder, _inputs| {
-                    binder.ensure_now_function_allowed()?;
-                    // NOTE: this will be further transformed during optimization. See the
-                    // documentation of `Now`.
-                    Ok(Now.into())
-                }),
-            )
-        }
-
-        fn pi() -> Handle {
-            raw_literal(ExprImpl::literal_f64(std::f64::consts::PI))
-        }
-
-        fn proctime() -> Handle {
-            Box::new(move |binder, inputs| {
-                binder.ensure_proctime_function_allowed()?;
-                raw_call(ExprType::Proctime)(binder, inputs)
-            })
-        }
-
-        // `SESSION_USER` is the user name of the user that is connected to the database.
-        fn session_user() -> Handle {
-            guard_by_len(
-                0,
-                raw(|binder, _inputs| {
-                    Ok(ExprImpl::literal_varchar(
-                        binder.auth_context.user_name.clone(),
-                    ))
-                }),
-            )
-        }
-
-        // `CURRENT_USER` is the user name of the user that is executing the command,
-        // `CURRENT_ROLE`, `USER` are synonyms for `CURRENT_USER`. Since we don't support
-        // `SET ROLE xxx` for now, they will all returns session user name.
-        fn current_user() -> Handle {
-            guard_by_len(
-                0,
-                raw(|binder, _inputs| {
-                    Ok(ExprImpl::literal_varchar(
-                        binder.auth_context.user_name.clone(),
-                    ))
-                }),
-            )
-        }
-
-        static HANDLES: LazyLock<HashMap<&'static str, Handle>> = LazyLock::new(|| {
-            [
-                (
-                    "booleq",
-                    rewrite(ExprType::Equal, Binder::rewrite_two_bool_inputs),
-                ),
-                (
-                    "boolne",
-                    rewrite(ExprType::NotEqual, Binder::rewrite_two_bool_inputs),
-                ),
-                ("coalesce", rewrite(ExprType::Coalesce, |inputs| {
-                    if inputs.iter().any(ExprImpl::has_table_function) {
-                        return Err(ErrorCode::BindError("table functions are not allowed in COALESCE".into()).into());
-                    }
-                    Ok(inputs)
-                })),
-                (
-                    "nullif",
-                    rewrite(ExprType::Case, Binder::rewrite_nullif_to_case_when),
-                ),
-                (
-                    "round",
-                    dispatch_by_len(vec![
-                        (2, raw_call(ExprType::RoundDigit)),
-                        (1, raw_call(ExprType::Round)),
-                    ]),
-                ),
-                ("pow", raw_call(ExprType::Pow)),
-                // "power" is the function name used in PG.
-                ("power", raw_call(ExprType::Pow)),
-                ("ceil", raw_call(ExprType::Ceil)),
-                ("ceiling", raw_call(ExprType::Ceil)),
-                ("floor", raw_call(ExprType::Floor)),
-                ("trunc", raw_call(ExprType::Trunc)),
-                ("abs", raw_call(ExprType::Abs)),
-                ("exp", raw_call(ExprType::Exp)),
-                ("ln", raw_call(ExprType::Ln)),
-                ("log", raw_call(ExprType::Log10)),
-                ("log10", raw_call(ExprType::Log10)),
-                ("mod", raw_call(ExprType::Modulus)),
-                ("sin", raw_call(ExprType::Sin)),
-                ("cos", raw_call(ExprType::Cos)),
-                ("tan", raw_call(ExprType::Tan)),
-                ("cot", raw_call(ExprType::Cot)),
-                ("asin", raw_call(ExprType::Asin)),
-                ("acos", raw_call(ExprType::Acos)),
-                ("atan", raw_call(ExprType::Atan)),
-                ("atan2", raw_call(ExprType::Atan2)),
-                ("sind", raw_call(ExprType::Sind)),
-                ("cosd", raw_call(ExprType::Cosd)),
-                ("cotd", raw_call(ExprType::Cotd)),
-                ("tand", raw_call(ExprType::Tand)),
-                ("sinh", raw_call(ExprType::Sinh)),
-                ("cosh", raw_call(ExprType::Cosh)),
-                ("tanh", raw_call(ExprType::Tanh)),
-                ("coth", raw_call(ExprType::Coth)),
-                ("asinh", raw_call(ExprType::Asinh)),
-                ("acosh", raw_call(ExprType::Acosh)),
-                ("atanh", raw_call(ExprType::Atanh)),
-                ("asind", raw_call(ExprType::Asind)),
-                ("degrees", raw_call(ExprType::Degrees)),
-                ("radians", raw_call(ExprType::Radians)),
-                ("sqrt", raw_call(ExprType::Sqrt)),
-                ("cbrt", raw_call(ExprType::Cbrt)),
-                ("sign", raw_call(ExprType::Sign)),
-                ("scale", raw_call(ExprType::Scale)),
-                ("min_scale", raw_call(ExprType::MinScale)),
-                ("trim_scale", raw_call(ExprType::TrimScale)),
-
-                (
-                    "to_timestamp",
-                    dispatch_by_len(vec![
-                        (1, raw_call(ExprType::SecToTimestamptz)),
-                        (2, raw_call(ExprType::CharToTimestamptz)),
-                    ]),
-                ),
-                ("date_trunc", raw_call(ExprType::DateTrunc)),
-                ("date_part", raw_call(ExprType::DatePart)),
-                ("make_date", raw_call(ExprType::MakeDate)),
-                ("make_time", raw_call(ExprType::MakeTime)),
-                ("make_timestamp", raw_call(ExprType::MakeTimestamp)),
-                ("to_date", raw_call(ExprType::CharToDate)),
-                ("make_timestamptz", raw_call(ExprType::MakeTimestamptz)),
-                // string
-                ("substr", raw_call(ExprType::Substr)),
-                ("length", raw_call(ExprType::Length)),
-                ("upper", raw_call(ExprType::Upper)),
-                ("lower", raw_call(ExprType::Lower)),
-                ("trim", raw_call(ExprType::Trim)),
-                ("replace", raw_call(ExprType::Replace)),
-                ("overlay", raw_call(ExprType::Overlay)),
-                ("btrim", raw_call(ExprType::Trim)),
-                ("ltrim", raw_call(ExprType::Ltrim)),
-                ("rtrim", raw_call(ExprType::Rtrim)),
-                ("md5", raw_call(ExprType::Md5)),
-                ("to_char", raw_call(ExprType::ToChar)),
-                (
-                    "concat",
-                    rewrite(ExprType::ConcatWs, Binder::rewrite_concat_to_concat_ws),
-                ),
-                ("concat_ws", raw_call(ExprType::ConcatWs)),
-                ("format", raw_call(ExprType::Format)),
-                ("translate", raw_call(ExprType::Translate)),
-                ("split_part", raw_call(ExprType::SplitPart)),
-                ("char_length", raw_call(ExprType::CharLength)),
-                ("character_length", raw_call(ExprType::CharLength)),
-                ("repeat", raw_call(ExprType::Repeat)),
-                ("ascii", raw_call(ExprType::Ascii)),
-                ("octet_length", raw_call(ExprType::OctetLength)),
-                ("bit_length", raw_call(ExprType::BitLength)),
-                ("regexp_match", raw_call(ExprType::RegexpMatch)),
-                ("regexp_replace", raw_call(ExprType::RegexpReplace)),
-                ("regexp_count", raw_call(ExprType::RegexpCount)),
-                ("regexp_split_to_array", raw_call(ExprType::RegexpSplitToArray)),
-                ("chr", raw_call(ExprType::Chr)),
-                ("starts_with", raw_call(ExprType::StartsWith)),
-                ("initcap", raw_call(ExprType::Initcap)),
-                ("lpad", raw_call(ExprType::Lpad)),
-                ("rpad", raw_call(ExprType::Rpad)),
-                ("reverse", raw_call(ExprType::Reverse)),
-                ("strpos", raw_call(ExprType::Position)),
-                ("to_ascii", raw_call(ExprType::ToAscii)),
-                ("to_hex", raw_call(ExprType::ToHex)),
-                ("quote_ident", raw_call(ExprType::QuoteIdent)),
-                ("quote_literal", guard_by_len(1, raw(|_binder, mut inputs| {
-                    if inputs[0].return_type() != DataType::Varchar {
-                        // Support `quote_literal(any)` by converting it to `quote_literal(any::text)`
-                        // Ref. https://github.com/postgres/postgres/blob/REL_16_1/src/include/catalog/pg_proc.dat#L4641
-                        FunctionCall::cast_mut(&mut inputs[0], DataType::Varchar, CastContext::Explicit)?;
-                    }
-                    Ok(FunctionCall::new_unchecked(ExprType::QuoteLiteral, inputs, DataType::Varchar).into())
-                }))),
-                ("quote_nullable", guard_by_len(1, raw(|_binder, mut inputs| {
-                    if inputs[0].return_type() != DataType::Varchar {
-                        // Support `quote_nullable(any)` by converting it to `quote_nullable(any::text)`
-                        // Ref. https://github.com/postgres/postgres/blob/REL_16_1/src/include/catalog/pg_proc.dat#L4650
-                        FunctionCall::cast_mut(&mut inputs[0], DataType::Varchar, CastContext::Explicit)?;
-                    }
-                    Ok(FunctionCall::new_unchecked(ExprType::QuoteNullable, inputs, DataType::Varchar).into())
-                }))),
-                ("string_to_array", raw_call(ExprType::StringToArray)),
-                ("encode", raw_call(ExprType::Encode)),
-                ("decode", raw_call(ExprType::Decode)),
-                ("convert_from", raw_call(ExprType::ConvertFrom)),
-                ("convert_to", raw_call(ExprType::ConvertTo)),
-                ("sha1", raw_call(ExprType::Sha1)),
-                ("sha224", raw_call(ExprType::Sha224)),
-                ("sha256", raw_call(ExprType::Sha256)),
-                ("sha384", raw_call(ExprType::Sha384)),
-                ("sha512", raw_call(ExprType::Sha512)),
-                ("encrypt", raw_call(ExprType::Encrypt)),
-                ("decrypt", raw_call(ExprType::Decrypt)),
-                ("left", raw_call(ExprType::Left)),
-                ("right", raw_call(ExprType::Right)),
-                ("inet_aton", raw_call(ExprType::InetAton)),
-                ("inet_ntoa", raw_call(ExprType::InetNtoa)),
-                ("int8send", raw_call(ExprType::PgwireSend)),
-                ("int8recv", guard_by_len(1, raw(|_binder, mut inputs| {
-                    // Similar to `cast` from string, return type is set explicitly rather than inferred.
-                    let hint = if !inputs[0].is_untyped() && inputs[0].return_type() == DataType::Varchar {
-                        " Consider `decode` or cast."
-                    } else {
-                        ""
-                    };
-                    inputs[0].cast_implicit_mut(DataType::Bytea).map_err(|e| {
-                        ErrorCode::BindError(format!("{} in `recv`.{hint}", e.as_report()))
-                    })?;
-                    Ok(FunctionCall::new_unchecked(ExprType::PgwireRecv, inputs, DataType::Int64).into())
-                }))),
-                // array
-                ("array_cat", raw_call(ExprType::ArrayCat)),
-                ("array_append", raw_call(ExprType::ArrayAppend)),
-                ("array_join", raw_call(ExprType::ArrayToString)),
-                ("array_prepend", raw_call(ExprType::ArrayPrepend)),
-                ("array_to_string", raw_call(ExprType::ArrayToString)),
-                ("array_distinct", raw_call(ExprType::ArrayDistinct)),
-                ("array_min", raw_call(ExprType::ArrayMin)),
-                ("array_sort", raw_call(ExprType::ArraySort)),
-                ("array_length", raw_call(ExprType::ArrayLength)),
-                ("cardinality", raw_call(ExprType::Cardinality)),
-                ("array_remove", raw_call(ExprType::ArrayRemove)),
-                ("array_replace", raw_call(ExprType::ArrayReplace)),
-                ("array_max", raw_call(ExprType::ArrayMax)),
-                ("array_sum", raw_call(ExprType::ArraySum)),
-                ("array_position", raw_call(ExprType::ArrayPosition)),
-                ("array_positions", raw_call(ExprType::ArrayPositions)),
-                ("array_contains", raw_call(ExprType::ArrayContains)),
-                ("arraycontains", raw_call(ExprType::ArrayContains)),
-                ("array_contained", raw_call(ExprType::ArrayContained)),
-                ("arraycontained", raw_call(ExprType::ArrayContained)),
-                ("trim_array", raw_call(ExprType::TrimArray)),
-                (
-                    "array_ndims",
-                    guard_by_len(1, raw(|_binder, inputs| {
-                        inputs[0].ensure_array_type()?;
-
-                        let n = inputs[0].return_type().array_ndims()
-                                .try_into().map_err(|_| ErrorCode::BindError("array_ndims integer overflow".into()))?;
-                        Ok(ExprImpl::literal_int(n))
-                    })),
-                ),
-                (
-                    "array_lower",
-                    guard_by_len(2, raw(|binder, inputs| {
-                        let (arg0, arg1) = inputs.into_iter().next_tuple().unwrap();
-                        // rewrite into `CASE WHEN 0 < arg1 AND arg1 <= array_ndims(arg0) THEN 1 END`
-                        let ndims_expr = binder.bind_builtin_scalar_function("array_ndims", vec![arg0], false)?;
-                        let arg1 = arg1.cast_implicit(DataType::Int32)?;
-
-                        FunctionCall::new(
-                            ExprType::Case,
-                            vec![
-                                FunctionCall::new(
-                                    ExprType::And,
-                                    vec![
-                                        FunctionCall::new(ExprType::LessThan, vec![ExprImpl::literal_int(0), arg1.clone()])?.into(),
-                                        FunctionCall::new(ExprType::LessThanOrEqual, vec![arg1, ndims_expr])?.into(),
-                                    ],
-                                )?.into(),
-                                ExprImpl::literal_int(1),
-                            ],
-                        ).map(Into::into)
-                    })),
-                ),
-                ("array_upper", raw_call(ExprType::ArrayLength)), // `lower == 1` implies `upper == length`
-                ("array_dims", raw_call(ExprType::ArrayDims)),
-                // int256
-                ("hex_to_int256", raw_call(ExprType::HexToInt256)),
-                // jsonb
-                ("jsonb_object_field", raw_call(ExprType::JsonbAccess)),
-                ("jsonb_array_element", raw_call(ExprType::JsonbAccess)),
-                ("jsonb_object_field_text", raw_call(ExprType::JsonbAccessStr)),
-                ("jsonb_array_element_text", raw_call(ExprType::JsonbAccessStr)),
-                ("jsonb_extract_path", raw_call(ExprType::JsonbExtractPath)),
-                ("jsonb_extract_path_text", raw_call(ExprType::JsonbExtractPathText)),
-                ("jsonb_typeof", raw_call(ExprType::JsonbTypeof)),
-                ("jsonb_array_length", raw_call(ExprType::JsonbArrayLength)),
-                ("jsonb_concat", raw_call(ExprType::JsonbConcat)),
-                ("jsonb_object", raw_call(ExprType::JsonbObject)),
-                ("jsonb_pretty", raw_call(ExprType::JsonbPretty)),
-                ("jsonb_contains", raw_call(ExprType::JsonbContains)),
-                ("jsonb_contained", raw_call(ExprType::JsonbContained)),
-                ("jsonb_exists", raw_call(ExprType::JsonbExists)),
-                ("jsonb_exists_any", raw_call(ExprType::JsonbExistsAny)),
-                ("jsonb_exists_all", raw_call(ExprType::JsonbExistsAll)),
-                ("jsonb_delete", raw_call(ExprType::Subtract)),
-                ("jsonb_delete_path", raw_call(ExprType::JsonbDeletePath)),
-                ("jsonb_strip_nulls", raw_call(ExprType::JsonbStripNulls)),
-                ("to_jsonb", raw_call(ExprType::ToJsonb)),
-                ("jsonb_build_array", raw_call(ExprType::JsonbBuildArray)),
-                ("jsonb_build_object", raw_call(ExprType::JsonbBuildObject)),
-                ("jsonb_populate_record", raw_call(ExprType::JsonbPopulateRecord)),
-                ("jsonb_path_match", raw_call(ExprType::JsonbPathMatch)),
-                ("jsonb_path_exists", raw_call(ExprType::JsonbPathExists)),
-                ("jsonb_path_query_array", raw_call(ExprType::JsonbPathQueryArray)),
-                ("jsonb_path_query_first", raw_call(ExprType::JsonbPathQueryFirst)),
-                ("jsonb_set", raw_call(ExprType::JsonbSet)),
-                // Functions that return a constant value
-                ("pi", pi()),
-                // greatest and least
-                ("greatest", raw_call(ExprType::Greatest)),
-                ("least", raw_call(ExprType::Least)),
-                // System information operations.
-                (
-                    "pg_typeof",
-                    guard_by_len(1, raw(|_binder, inputs| {
-                        let input = &inputs[0];
-                        let v = match input.is_untyped() {
-                            true => "unknown".into(),
-                            false => input.return_type().to_string(),
-                        };
-                        Ok(ExprImpl::literal_varchar(v))
-                    })),
-                ),
-                ("current_database", guard_by_len(0, raw(|binder, _inputs| {
-                    Ok(ExprImpl::literal_varchar(binder.db_name.clone()))
-                }))),
-                ("current_schema", guard_by_len(0, raw(|binder, _inputs| {
-                    return Ok(binder
-                        .first_valid_schema()
-                        .map(|schema| ExprImpl::literal_varchar(schema.name()))
-                        .unwrap_or_else(|_| ExprImpl::literal_null(DataType::Varchar)));
-                }))),
-                ("current_schemas", raw(|binder, mut inputs| {
-                    let no_match_err = ErrorCode::ExprError(
-                            "No function matches the given name and argument types. You might need to add explicit type casts.".into()
-                        );
-                    if inputs.len() != 1 {
-                        return Err(no_match_err.into());
-                    }
-                    let input = inputs
-                        .pop()
-                        .unwrap()
-                        .enforce_bool_clause("current_schemas")
-                        .map_err(|_| no_match_err)?;
-
-                    let ExprImpl::Literal(literal) = &input else {
-                        bail_not_implemented!("Only boolean literals are supported in `current_schemas`.");
-                    };
-
-                    let Some(bool) = literal.get_data().as_ref().map(|bool| bool.clone().into_bool()) else {
-                        return Ok(ExprImpl::literal_null(DataType::List(Box::new(DataType::Varchar))));
-                    };
-
-                    let paths = if bool {
-                        binder.search_path.path()
-                    } else {
-                        binder.search_path.real_path()
-                    };
-
-                    let mut schema_names = vec![];
-                    for path in paths {
-                        let mut schema_name = path;
-                        if schema_name == USER_NAME_WILD_CARD {
-                            schema_name = &binder.auth_context.user_name;
-                        }
-
-                        if binder
-                            .catalog
-                            .get_schema_by_name(&binder.db_name, schema_name)
-                            .is_ok()
-                        {
-                            schema_names.push(schema_name.as_str());
-                        }
-                    }
-
-                    Ok(ExprImpl::literal_list(
-                        ListValue::from_iter(schema_names),
-                        DataType::Varchar,
-                    ))
-                })),
-                ("session_user", session_user()),
-                ("current_role", current_user()),
-                ("current_user", current_user()),
-                ("user", current_user()),
-                ("pg_get_userbyid", raw_call(ExprType::PgGetUserbyid)),
-                ("pg_get_indexdef", raw_call(ExprType::PgGetIndexdef)),
-                ("pg_get_viewdef", raw_call(ExprType::PgGetViewdef)),
-                ("pg_index_column_has_property", raw_call(ExprType::PgIndexColumnHasProperty)),
-                ("pg_relation_size", raw(|_binder, mut inputs|{
-                    if inputs.is_empty() {
-                        return Err(ErrorCode::ExprError(
-                            "function pg_relation_size() does not exist".into(),
-                        )
-                        .into());
-                    }
-                    inputs[0].cast_to_regclass_mut()?;
-                    Ok(FunctionCall::new(ExprType::PgRelationSize, inputs)?.into())
-                })),
-                ("pg_get_serial_sequence", raw_literal(ExprImpl::literal_null(DataType::Varchar))),
-                ("pg_table_size", guard_by_len(1, raw(|_binder, mut inputs|{
-                    inputs[0].cast_to_regclass_mut()?;
-                    Ok(FunctionCall::new(ExprType::PgRelationSize, inputs)?.into())
-                }))),
-                ("pg_indexes_size", guard_by_len(1, raw(|_binder, mut inputs|{
-                    inputs[0].cast_to_regclass_mut()?;
-                    Ok(FunctionCall::new(ExprType::PgIndexesSize, inputs)?.into())
-                }))),
-                ("pg_get_expr", raw(|_binder, inputs|{
-                    if inputs.len() == 2 || inputs.len() == 3 {
-                        // TODO: implement pg_get_expr rather than just return empty as an workaround.
-                        Ok(ExprImpl::literal_varchar("".into()))
-                    } else {
-                        Err(ErrorCode::ExprError(
-                            "Too many/few arguments for pg_catalog.pg_get_expr()".into(),
-                        )
-                        .into())
-                    }
-                })),
-                ("current_setting", guard_by_len(1, raw(|binder, inputs| {
-                    let input = &inputs[0];
-                    let input = if let ExprImpl::Literal(literal) = input &&
-                        let Some(ScalarImpl::Utf8(input)) = literal.get_data()
-                    {
-                        input
-                    } else {
-                        return Err(ErrorCode::ExprError(
-                            "Only literal is supported in `setting_name`.".into(),
-                        )
-                        .into());
-                    };
-                    let session_config = binder.session_config.read();
-                    Ok(ExprImpl::literal_varchar(session_config.get(input.as_ref())?))
-                }))),
-                ("set_config", guard_by_len(3, raw(|binder, inputs| {
-                    let setting_name = if let ExprImpl::Literal(literal) = &inputs[0] && let Some(ScalarImpl::Utf8(input)) = literal.get_data() {
-                        input
-                    } else {
-                        return Err(ErrorCode::ExprError(
-                            "Only string literal is supported in `setting_name`.".into(),
-                        )
-                        .into());
-                    };
-
-                    let new_value = if let ExprImpl::Literal(literal) = &inputs[1] && let Some(ScalarImpl::Utf8(input)) = literal.get_data() {
-                        input
-                    } else {
-                        return Err(ErrorCode::ExprError(
-                            "Only string literal is supported in `setting_name`.".into(),
-                        )
-                        .into());
-                    };
-
-                    let is_local = if let ExprImpl::Literal(literal) = &inputs[2] && let Some(ScalarImpl::Bool(input)) = literal.get_data() {
-                        input
-                    } else {
-                        return Err(ErrorCode::ExprError(
-                            "Only bool literal is supported in `is_local`.".into(),
-                        )
-                        .into());
-                    };
-
-                    if *is_local {
-                        return Err(ErrorCode::ExprError(
-                            "`is_local = true` is not supported now.".into(),
-                        )
-                        .into());
-                    }
-
-                    let mut session_config = binder.session_config.write();
-
-                    // TODO: report session config changes if necessary.
-                    session_config.set(setting_name, new_value.to_string(), &mut())?;
-
-                    Ok(ExprImpl::literal_varchar(new_value.to_string()))
-                }))),
-                ("format_type", raw_call(ExprType::FormatType)),
-                ("pg_table_is_visible", raw_literal(ExprImpl::literal_bool(true))),
-                ("pg_type_is_visible", raw_literal(ExprImpl::literal_bool(true))),
-                ("pg_get_constraintdef", raw_literal(ExprImpl::literal_null(DataType::Varchar))),
-                ("pg_get_partkeydef", raw_literal(ExprImpl::literal_null(DataType::Varchar))),
-                ("pg_encoding_to_char", raw_literal(ExprImpl::literal_varchar("UTF8".into()))),
-                ("has_database_privilege", raw_literal(ExprImpl::literal_bool(true))),
-                ("has_table_privilege", raw(|binder, mut inputs|{
-                    if inputs.len() == 2 {
-                        inputs.insert(0, ExprImpl::literal_varchar(binder.auth_context.user_name.clone()));
-                    }
-                    if inputs.len() == 3 {
-                        if inputs[1].return_type() == DataType::Varchar {
-                            inputs[1].cast_to_regclass_mut()?;
-                        }
-                        Ok(FunctionCall::new(ExprType::HasTablePrivilege, inputs)?.into())
-                    } else {
-                        Err(ErrorCode::ExprError(
-                            "Too many/few arguments for pg_catalog.has_table_privilege()".into(),
-                        )
-                        .into())
-                    }
-                })),
-                ("has_any_column_privilege", raw(|binder, mut inputs|{
-                    if inputs.len() == 2 {
-                        inputs.insert(0, ExprImpl::literal_varchar(binder.auth_context.user_name.clone()));
-                    }
-                    if inputs.len() == 3 {
-                        if inputs[1].return_type() == DataType::Varchar {
-                            inputs[1].cast_to_regclass_mut()?;
-                        }
-                        Ok(FunctionCall::new(ExprType::HasAnyColumnPrivilege, inputs)?.into())
-                    } else {
-                        Err(ErrorCode::ExprError(
-                            "Too many/few arguments for pg_catalog.has_any_column_privilege()".into(),
-                        )
-                        .into())
-                    }
-                })),
-                ("has_schema_privilege", raw(|binder, mut inputs|{
-                    if inputs.len() == 2 {
-                        inputs.insert(0, ExprImpl::literal_varchar(binder.auth_context.user_name.clone()));
-                    }
-                    if inputs.len() == 3 {
-                        Ok(FunctionCall::new(ExprType::HasSchemaPrivilege, inputs)?.into())
-                    } else {
-                        Err(ErrorCode::ExprError(
-                            "Too many/few arguments for pg_catalog.has_schema_privilege()".into(),
-                        )
-                        .into())
-                    }
-                })),
-                ("pg_stat_get_numscans", raw_literal(ExprImpl::literal_bigint(0))),
-                ("pg_backend_pid", raw(|binder, _inputs| {
-                    // FIXME: the session id is not global unique in multi-frontend env.
-                    Ok(ExprImpl::literal_int(binder.session_id.0))
-                })),
-                ("pg_cancel_backend", guard_by_len(1, raw(|_binder, _inputs| {
-                        // TODO: implement real cancel rather than just return false as an workaround.
-                        Ok(ExprImpl::literal_bool(false))
-                }))),
-                ("pg_terminate_backend", guard_by_len(1, raw(|_binder, _inputs|{
-                        // TODO: implement real terminate rather than just return false as an
-                        // workaround.
-                        Ok(ExprImpl::literal_bool(false))
-                }))),
-                ("pg_tablespace_location", guard_by_len(1, raw_literal(ExprImpl::literal_null(DataType::Varchar)))),
-                ("pg_postmaster_start_time", guard_by_len(0, raw(|_binder, _inputs|{
-                    let server_start_time = risingwave_variables::get_server_start_time();
-                    let datum = server_start_time.map(Timestamptz::from).map(ScalarImpl::from);
-                    let literal = Literal::new(datum, DataType::Timestamptz);
-                    Ok(literal.into())
-                }))),
-                // TODO: really implement them.
-                // https://www.postgresql.org/docs/9.5/functions-info.html#FUNCTIONS-INFO-COMMENT-TABLE
-                // WARN: Hacked in [`Binder::bind_function`]!!!
-                ("col_description", raw_call(ExprType::ColDescription)),
-                ("obj_description", raw_literal(ExprImpl::literal_varchar("".to_string()))),
-                ("shobj_description", raw_literal(ExprImpl::literal_varchar("".to_string()))),
-                ("pg_is_in_recovery", raw_literal(ExprImpl::literal_bool(false))),
-                // internal
-                ("rw_vnode", raw_call(ExprType::Vnode)),
-                ("rw_test_paid_tier", raw_call(ExprType::TestPaidTier)), // for testing purposes
-                // TODO: choose which pg version we should return.
-                ("version", raw_literal(ExprImpl::literal_varchar(current_cluster_version()))),
-                // non-deterministic
-                ("now", now()),
-                ("current_timestamp", now()),
-                ("proctime", proctime()),
-                ("pg_sleep", raw_call(ExprType::PgSleep)),
-                ("pg_sleep_for", raw_call(ExprType::PgSleepFor)),
-                // TODO: implement pg_sleep_until
-                // ("pg_sleep_until", raw_call(ExprType::PgSleepUntil)),
-
-                // cast functions
-                // only functions required by the existing PostgreSQL tool are implemented
-                ("date", guard_by_len(1, raw(|_binder, inputs| {
-                    inputs[0].clone().cast_explicit(DataType::Date).map_err(Into::into)
-                }))),
-            ]
-            .into_iter()
-            .collect()
-        });
-
-        static FUNCTIONS_BKTREE: LazyLock<BKTree<&str>> = LazyLock::new(|| {
-            let mut tree = BKTree::new(metrics::Levenshtein);
-
-            // TODO: Also hint other functinos, e.g., Agg or UDF.
-            for k in HANDLES.keys() {
-                tree.add(*k);
-            }
-
-            tree
-        });
-
-        if variadic {
-            let func = match function_name {
-                "format" => ExprType::FormatVariadic,
-                "concat" => ExprType::ConcatVariadic,
-                "concat_ws" => ExprType::ConcatWsVariadic,
-                "jsonb_build_array" => ExprType::JsonbBuildArrayVariadic,
-                "jsonb_build_object" => ExprType::JsonbBuildObjectVariadic,
-                "jsonb_extract_path" => ExprType::JsonbExtractPathVariadic,
-                "jsonb_extract_path_text" => ExprType::JsonbExtractPathTextVariadic,
-                _ => {
-                    return Err(ErrorCode::BindError(format!(
-                        "VARIADIC argument is not allowed in function \"{}\"",
-                        function_name
-                    ))
-                    .into())
-                }
-            };
-            return Ok(FunctionCall::new(func, inputs)?.into());
-        }
-
-        match HANDLES.get(function_name) {
-            Some(handle) => handle(self, inputs),
-            None => {
-                let allowed_distance = if function_name.len() > 3 { 2 } else { 1 };
-
-                let candidates = FUNCTIONS_BKTREE
-                    .find(function_name, allowed_distance)
-                    .map(|(_idx, c)| c)
-                    .join(" or ");
-
-                Err(no_function!(
-                    candidates = (!candidates.is_empty()).then_some(candidates),
-                    "{}({})",
-                    function_name,
-                    inputs.iter().map(|e| e.return_type()).join(", ")
-                )
-                .into())
-            }
-        }
-    }
-
-    fn rewrite_concat_to_concat_ws(inputs: Vec<ExprImpl>) -> Result<Vec<ExprImpl>> {
-        if inputs.is_empty() {
-            Err(ErrorCode::BindError(
-                "Function `concat` takes at least 1 arguments (0 given)".to_string(),
-            )
-            .into())
-        } else {
-            let inputs = once(ExprImpl::literal_varchar("".to_string()))
-                .chain(inputs)
-                .collect();
-            Ok(inputs)
-        }
-    }
-
-    /// Make sure inputs only have 2 value and rewrite the arguments.
-    /// Nullif(expr1,expr2) -> Case(Equal(expr1 = expr2),null,expr1).
-    fn rewrite_nullif_to_case_when(inputs: Vec<ExprImpl>) -> Result<Vec<ExprImpl>> {
-        if inputs.len() != 2 {
-            Err(
-                ErrorCode::BindError("Function `nullif` must contain 2 arguments".to_string())
-                    .into(),
-            )
-        } else {
-            let inputs = vec![
-                FunctionCall::new(ExprType::Equal, inputs.clone())?.into(),
-                Literal::new(None, inputs[0].return_type()).into(),
-                inputs[0].clone(),
-            ];
-            Ok(inputs)
-        }
-    }
-
-    fn rewrite_two_bool_inputs(mut inputs: Vec<ExprImpl>) -> Result<Vec<ExprImpl>> {
-        if inputs.len() != 2 {
-            return Err(
-                ErrorCode::BindError("function must contain only 2 arguments".to_string()).into(),
-            );
-        }
-        let left = inputs.pop().unwrap();
-        let right = inputs.pop().unwrap();
-        Ok(vec![
-            left.cast_implicit(DataType::Boolean)?,
-            right.cast_implicit(DataType::Boolean)?,
-        ])
-    }
-
-    fn ensure_window_function_allowed(&self) -> Result<()> {
-        if let Some(clause) = self.context.clause {
-            match clause {
-                Clause::Where
-                | Clause::Values
-                | Clause::GroupBy
-                | Clause::Having
-                | Clause::Filter
-                | Clause::GeneratedColumn
-                | Clause::From
-                | Clause::Insert
-                | Clause::JoinOn => {
-                    return Err(ErrorCode::InvalidInputSyntax(format!(
-                        "window functions are not allowed in {}",
-                        clause
-                    ))
-                    .into());
-                }
-            }
-        }
-        Ok(())
-    }
-
-    fn ensure_now_function_allowed(&self) -> Result<()> {
-        if self.is_for_stream()
-            && !matches!(
-                self.context.clause,
-                Some(Clause::Where)
-                    | Some(Clause::Having)
-                    | Some(Clause::JoinOn)
-                    | Some(Clause::From)
-            )
-        {
-            return Err(ErrorCode::InvalidInputSyntax(format!(
-                "For streaming queries, `NOW()` function is only allowed in `WHERE`, `HAVING`, `ON` and `FROM`. Found in clause: {:?}. \
-                Please please refer to https://www.risingwave.dev/docs/current/sql-pattern-temporal-filters/ for more information",
-                self.context.clause
-            ))
-            .into());
-        }
-        if matches!(self.context.clause, Some(Clause::GeneratedColumn)) {
-            return Err(ErrorCode::InvalidInputSyntax(
-                "Cannot use `NOW()` function in generated columns. Do you want `PROCTIME()`?"
-                    .to_string(),
-            )
-            .into());
-        }
-        Ok(())
-    }
-
-    fn ensure_proctime_function_allowed(&self) -> Result<()> {
-        if !self.is_for_ddl() {
-            return Err(ErrorCode::InvalidInputSyntax(
-                "Function `PROCTIME()` is only allowed in CREATE TABLE/SOURCE. Is `NOW()` what you want?".to_string(),
-            )
-            .into());
-        }
-        Ok(())
-    }
-
-    fn ensure_aggregate_allowed(&self) -> Result<()> {
-        if let Some(clause) = self.context.clause {
-            match clause {
-                Clause::Where
-                | Clause::Values
-                | Clause::From
-                | Clause::GeneratedColumn
-                | Clause::Insert
-                | Clause::JoinOn => {
-                    return Err(ErrorCode::InvalidInputSyntax(format!(
-                        "aggregate functions are not allowed in {}",
-                        clause
-                    ))
-                    .into())
-                }
-                Clause::Having | Clause::Filter | Clause::GroupBy => {}
-            }
-        }
-        Ok(())
-    }
-
-    fn ensure_table_function_allowed(&self) -> Result<()> {
-        if let Some(clause) = self.context.clause {
-            match clause {
-                Clause::JoinOn
-                | Clause::Where
-                | Clause::Having
-                | Clause::Filter
-                | Clause::Values
-                | Clause::Insert
-                | Clause::GeneratedColumn => {
-                    return Err(ErrorCode::InvalidInputSyntax(format!(
-                        "table functions are not allowed in {}",
-                        clause
-                    ))
-                    .into());
-                }
-                Clause::GroupBy | Clause::From => {}
-            }
-        }
-        Ok(())
-    }
-
-    pub(in crate::binder) fn bind_function_expr_arg(
-        &mut self,
-        arg_expr: FunctionArgExpr,
-    ) -> Result<Vec<ExprImpl>> {
-        match arg_expr {
-            FunctionArgExpr::Expr(expr) => Ok(vec![self.bind_expr_inner(expr)?]),
-            FunctionArgExpr::QualifiedWildcard(_, _)
-            | FunctionArgExpr::ExprQualifiedWildcard(_, _) => Err(ErrorCode::InvalidInputSyntax(
-                format!("unexpected wildcard {}", arg_expr),
-            )
-            .into()),
-            FunctionArgExpr::Wildcard(None) => Ok(vec![]),
-            FunctionArgExpr::Wildcard(Some(_)) => unreachable!(),
-        }
-    }
-
-    pub(in crate::binder) fn bind_function_arg(
-        &mut self,
-        arg: FunctionArg,
-    ) -> Result<Vec<ExprImpl>> {
-        match arg {
-            FunctionArg::Unnamed(expr) => self.bind_function_expr_arg(expr),
-            FunctionArg::Named { .. } => todo!(),
-        }
-    }
-}
diff --git a/src/frontend/src/binder/expr/function/aggregate.rs b/src/frontend/src/binder/expr/function/aggregate.rs
new file mode 100644
index 0000000000000..77538b799bad2
--- /dev/null
+++ b/src/frontend/src/binder/expr/function/aggregate.rs
@@ -0,0 +1,294 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use itertools::Itertools;
+use risingwave_common::bail_not_implemented;
+use risingwave_common::types::DataType;
+use risingwave_expr::aggregate::{agg_kinds, AggKind, PbAggKind};
+use risingwave_sqlparser::ast::{Function, FunctionArgExpr};
+
+use crate::binder::Clause;
+use crate::error::{ErrorCode, Result};
+use crate::expr::{AggCall, ExprImpl, Literal, OrderBy};
+use crate::utils::Condition;
+use crate::Binder;
+
+impl Binder {
+    fn ensure_aggregate_allowed(&self) -> Result<()> {
+        if let Some(clause) = self.context.clause {
+            match clause {
+                Clause::Where
+                | Clause::Values
+                | Clause::From
+                | Clause::GeneratedColumn
+                | Clause::Insert
+                | Clause::JoinOn => {
+                    return Err(ErrorCode::InvalidInputSyntax(format!(
+                        "aggregate functions are not allowed in {}",
+                        clause
+                    ))
+                    .into())
+                }
+                Clause::Having | Clause::Filter | Clause::GroupBy => {}
+            }
+        }
+        Ok(())
+    }
+
+    pub(super) fn bind_aggregate_function(
+        &mut self,
+        f: Function,
+        kind: AggKind,
+    ) -> Result<ExprImpl> {
+        self.ensure_aggregate_allowed()?;
+
+        let distinct = f.distinct;
+        let filter_expr = f.filter.clone();
+
+        let (direct_args, args, order_by) = if matches!(kind, agg_kinds::ordered_set!()) {
+            self.bind_ordered_set_agg(f, kind.clone())?
+        } else {
+            self.bind_normal_agg(f, kind.clone())?
+        };
+
+        let filter = match filter_expr {
+            Some(filter) => {
+                let mut clause = Some(Clause::Filter);
+                std::mem::swap(&mut self.context.clause, &mut clause);
+                let expr = self
+                    .bind_expr_inner(*filter)
+                    .and_then(|expr| expr.enforce_bool_clause("FILTER"))?;
+                self.context.clause = clause;
+                if expr.has_subquery() {
+                    bail_not_implemented!("subquery in filter clause");
+                }
+                if expr.has_agg_call() {
+                    bail_not_implemented!("aggregation function in filter clause");
+                }
+                if expr.has_table_function() {
+                    bail_not_implemented!("table function in filter clause");
+                }
+                Condition::with_expr(expr)
+            }
+            None => Condition::true_cond(),
+        };
+
+        Ok(ExprImpl::AggCall(Box::new(AggCall::new(
+            kind,
+            args,
+            distinct,
+            order_by,
+            filter,
+            direct_args,
+        )?)))
+    }
+
+    fn bind_ordered_set_agg(
+        &mut self,
+        f: Function,
+        kind: AggKind,
+    ) -> Result<(Vec<Literal>, Vec<ExprImpl>, OrderBy)> {
+        // Syntax:
+        // aggregate_name ( [ expression [ , ... ] ] ) WITHIN GROUP ( order_by_clause ) [ FILTER
+        // ( WHERE filter_clause ) ]
+
+        assert!(matches!(kind, agg_kinds::ordered_set!()));
+
+        if !f.order_by.is_empty() {
+            return Err(ErrorCode::InvalidInputSyntax(format!(
+                "ORDER BY is not allowed for ordered-set aggregation `{}`",
+                kind
+            ))
+            .into());
+        }
+        if f.distinct {
+            return Err(ErrorCode::InvalidInputSyntax(format!(
+                "DISTINCT is not allowed for ordered-set aggregation `{}`",
+                kind
+            ))
+            .into());
+        }
+
+        let within_group = *f.within_group.ok_or_else(|| {
+            ErrorCode::InvalidInputSyntax(format!(
+                "WITHIN GROUP is expected for ordered-set aggregation `{}`",
+                kind
+            ))
+        })?;
+
+        let mut direct_args: Vec<_> = f
+            .args
+            .into_iter()
+            .map(|arg| self.bind_function_arg(arg))
+            .flatten_ok()
+            .try_collect()?;
+        let mut args =
+            self.bind_function_expr_arg(FunctionArgExpr::Expr(within_group.expr.clone()))?;
+        let order_by = OrderBy::new(vec![self.bind_order_by_expr(within_group)?]);
+
+        // check signature and do implicit cast
+        match (&kind, direct_args.as_mut_slice(), args.as_mut_slice()) {
+            (
+                AggKind::Builtin(PbAggKind::PercentileCont | PbAggKind::PercentileDisc),
+                [fraction],
+                [arg],
+            ) => {
+                decimal_to_float64(fraction, &kind)?;
+                if matches!(&kind, AggKind::Builtin(PbAggKind::PercentileCont)) {
+                    arg.cast_implicit_mut(DataType::Float64).map_err(|_| {
+                        ErrorCode::InvalidInputSyntax(format!(
+                            "arg in `{}` must be castable to float64",
+                            kind
+                        ))
+                    })?;
+                }
+            }
+            (AggKind::Builtin(PbAggKind::Mode), [], [_arg]) => {}
+            (
+                AggKind::Builtin(PbAggKind::ApproxPercentile),
+                [percentile, relative_error],
+                [_percentile_col],
+            ) => {
+                decimal_to_float64(percentile, &kind)?;
+                decimal_to_float64(relative_error, &kind)?;
+            }
+            _ => {
+                return Err(ErrorCode::InvalidInputSyntax(format!(
+                    "invalid direct args or within group argument for `{}` aggregation",
+                    kind
+                ))
+                .into())
+            }
+        }
+
+        Ok((
+            direct_args
+                .into_iter()
+                .map(|arg| *arg.into_literal().unwrap())
+                .collect(),
+            args,
+            order_by,
+        ))
+    }
+
+    fn bind_normal_agg(
+        &mut self,
+        f: Function,
+        kind: AggKind,
+    ) -> Result<(Vec<Literal>, Vec<ExprImpl>, OrderBy)> {
+        // Syntax:
+        // aggregate_name (expression [ , ... ] [ order_by_clause ] ) [ FILTER ( WHERE
+        //   filter_clause ) ]
+        // aggregate_name (ALL expression [ , ... ] [ order_by_clause ] ) [ FILTER ( WHERE
+        //   filter_clause ) ]
+        // aggregate_name (DISTINCT expression [ , ... ] [ order_by_clause ] ) [ FILTER ( WHERE
+        //   filter_clause ) ]
+        // aggregate_name ( * ) [ FILTER ( WHERE filter_clause ) ]
+
+        assert!(!matches!(kind, agg_kinds::ordered_set!()));
+
+        if f.within_group.is_some() {
+            return Err(ErrorCode::InvalidInputSyntax(format!(
+                "WITHIN GROUP is not allowed for non-ordered-set aggregation `{}`",
+                kind
+            ))
+            .into());
+        }
+
+        let args: Vec<_> = f
+            .args
+            .iter()
+            .map(|arg| self.bind_function_arg(arg.clone()))
+            .flatten_ok()
+            .try_collect()?;
+        let order_by = OrderBy::new(
+            f.order_by
+                .into_iter()
+                .map(|e| self.bind_order_by_expr(e))
+                .try_collect()?,
+        );
+
+        if f.distinct {
+            if matches!(
+                kind,
+                AggKind::Builtin(PbAggKind::ApproxCountDistinct)
+                    | AggKind::Builtin(PbAggKind::ApproxPercentile)
+            ) {
+                return Err(ErrorCode::InvalidInputSyntax(format!(
+                    "DISTINCT is not allowed for approximate aggregation `{}`",
+                    kind
+                ))
+                .into());
+            }
+
+            if args.is_empty() {
+                return Err(ErrorCode::InvalidInputSyntax(format!(
+                    "DISTINCT is not allowed for aggregate function `{}` without args",
+                    kind
+                ))
+                .into());
+            }
+
+            // restrict arguments[1..] to be constant because we don't support multiple distinct key
+            // indices for now
+            if args.iter().skip(1).any(|arg| arg.as_literal().is_none()) {
+                bail_not_implemented!("non-constant arguments other than the first one for DISTINCT aggregation is not supported now");
+            }
+
+            // restrict ORDER BY to align with PG, which says:
+            // > If DISTINCT is specified in addition to an order_by_clause, then all the ORDER BY
+            // > expressions must match regular arguments of the aggregate; that is, you cannot sort
+            // > on an expression that is not included in the DISTINCT list.
+            if !order_by.sort_exprs.iter().all(|e| args.contains(&e.expr)) {
+                return Err(ErrorCode::InvalidInputSyntax(format!(
+                    "ORDER BY expressions must match regular arguments of the aggregate for `{}` when DISTINCT is provided",
+                    kind
+                ))
+                .into());
+            }
+        }
+
+        Ok((vec![], args, order_by))
+    }
+}
+
+fn decimal_to_float64(decimal_expr: &mut ExprImpl, kind: &AggKind) -> Result<()> {
+    if decimal_expr.cast_implicit_mut(DataType::Float64).is_err() {
+        return Err(ErrorCode::InvalidInputSyntax(format!(
+            "direct arg in `{}` must be castable to float64",
+            kind
+        ))
+        .into());
+    }
+
+    let Some(Ok(fraction_datum)) = decimal_expr.try_fold_const() else {
+        bail_not_implemented!(
+            issue = 14079,
+            "variable as direct argument of ordered-set aggregate",
+        );
+    };
+
+    if let Some(ref fraction_value) = fraction_datum
+        && !(0.0..=1.0).contains(&fraction_value.as_float64().0)
+    {
+        return Err(ErrorCode::InvalidInputSyntax(format!(
+            "direct arg in `{}` must between 0.0 and 1.0",
+            kind
+        ))
+        .into());
+    }
+    // note that the fraction can be NULL
+    *decimal_expr = Literal::new(fraction_datum, DataType::Float64).into();
+    Ok(())
+}
diff --git a/src/frontend/src/binder/expr/function/builtin_scalar.rs b/src/frontend/src/binder/expr/function/builtin_scalar.rs
new file mode 100644
index 0000000000000..b38a36586d1e7
--- /dev/null
+++ b/src/frontend/src/binder/expr/function/builtin_scalar.rs
@@ -0,0 +1,795 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::HashMap;
+use std::sync::LazyLock;
+
+use bk_tree::{metrics, BKTree};
+use itertools::Itertools;
+use risingwave_common::session_config::USER_NAME_WILD_CARD;
+use risingwave_common::types::{DataType, ListValue, ScalarImpl, Timestamptz};
+use risingwave_common::{bail_not_implemented, current_cluster_version, no_function};
+use thiserror_ext::AsReport;
+
+use crate::binder::Clause;
+use crate::error::{ErrorCode, Result};
+use crate::expr::{CastContext, Expr, ExprImpl, ExprType, FunctionCall, Literal, Now};
+use crate::Binder;
+
+impl Binder {
+    pub(super) fn bind_builtin_scalar_function(
+        &mut self,
+        function_name: &str,
+        inputs: Vec<ExprImpl>,
+        variadic: bool,
+    ) -> Result<ExprImpl> {
+        type Inputs = Vec<ExprImpl>;
+
+        type Handle = Box<dyn Fn(&mut Binder, Inputs) -> Result<ExprImpl> + Sync + Send>;
+
+        fn rewrite(r#type: ExprType, rewriter: fn(Inputs) -> Result<Inputs>) -> Handle {
+            Box::new(move |_binder, mut inputs| {
+                inputs = (rewriter)(inputs)?;
+                Ok(FunctionCall::new(r#type, inputs)?.into())
+            })
+        }
+
+        fn raw_call(r#type: ExprType) -> Handle {
+            rewrite(r#type, Ok)
+        }
+
+        fn guard_by_len(expected_len: usize, handle: Handle) -> Handle {
+            Box::new(move |binder, inputs| {
+                if inputs.len() == expected_len {
+                    handle(binder, inputs)
+                } else {
+                    Err(ErrorCode::ExprError("unexpected arguments number".into()).into())
+                }
+            })
+        }
+
+        fn raw<F: Fn(&mut Binder, Inputs) -> Result<ExprImpl> + Sync + Send + 'static>(
+            f: F,
+        ) -> Handle {
+            Box::new(f)
+        }
+
+        fn dispatch_by_len(mapping: Vec<(usize, Handle)>) -> Handle {
+            Box::new(move |binder, inputs| {
+                for (len, handle) in &mapping {
+                    if inputs.len() == *len {
+                        return handle(binder, inputs);
+                    }
+                }
+                Err(ErrorCode::ExprError("unexpected arguments number".into()).into())
+            })
+        }
+
+        fn raw_literal(literal: ExprImpl) -> Handle {
+            Box::new(move |_binder, _inputs| Ok(literal.clone()))
+        }
+
+        fn now() -> Handle {
+            guard_by_len(
+                0,
+                raw(move |binder, _inputs| {
+                    binder.ensure_now_function_allowed()?;
+                    // NOTE: this will be further transformed during optimization. See the
+                    // documentation of `Now`.
+                    Ok(Now.into())
+                }),
+            )
+        }
+
+        fn pi() -> Handle {
+            raw_literal(ExprImpl::literal_f64(std::f64::consts::PI))
+        }
+
+        fn proctime() -> Handle {
+            Box::new(move |binder, inputs| {
+                binder.ensure_proctime_function_allowed()?;
+                raw_call(ExprType::Proctime)(binder, inputs)
+            })
+        }
+
+        // `SESSION_USER` is the user name of the user that is connected to the database.
+        fn session_user() -> Handle {
+            guard_by_len(
+                0,
+                raw(|binder, _inputs| {
+                    Ok(ExprImpl::literal_varchar(
+                        binder.auth_context.user_name.clone(),
+                    ))
+                }),
+            )
+        }
+
+        // `CURRENT_USER` is the user name of the user that is executing the command,
+        // `CURRENT_ROLE`, `USER` are synonyms for `CURRENT_USER`. Since we don't support
+        // `SET ROLE xxx` for now, they will all returns session user name.
+        fn current_user() -> Handle {
+            guard_by_len(
+                0,
+                raw(|binder, _inputs| {
+                    Ok(ExprImpl::literal_varchar(
+                        binder.auth_context.user_name.clone(),
+                    ))
+                }),
+            )
+        }
+
+        static HANDLES: LazyLock<HashMap<&'static str, Handle>> = LazyLock::new(|| {
+            [
+                (
+                    "booleq",
+                    rewrite(ExprType::Equal, rewrite_two_bool_inputs),
+                ),
+                (
+                    "boolne",
+                    rewrite(ExprType::NotEqual, rewrite_two_bool_inputs),
+                ),
+                ("coalesce", rewrite(ExprType::Coalesce, |inputs| {
+                    if inputs.iter().any(ExprImpl::has_table_function) {
+                        return Err(ErrorCode::BindError("table functions are not allowed in COALESCE".into()).into());
+                    }
+                    Ok(inputs)
+                })),
+                (
+                    "nullif",
+                    rewrite(ExprType::Case, rewrite_nullif_to_case_when),
+                ),
+                (
+                    "round",
+                    dispatch_by_len(vec![
+                        (2, raw_call(ExprType::RoundDigit)),
+                        (1, raw_call(ExprType::Round)),
+                    ]),
+                ),
+                ("pow", raw_call(ExprType::Pow)),
+                // "power" is the function name used in PG.
+                ("power", raw_call(ExprType::Pow)),
+                ("ceil", raw_call(ExprType::Ceil)),
+                ("ceiling", raw_call(ExprType::Ceil)),
+                ("floor", raw_call(ExprType::Floor)),
+                ("trunc", raw_call(ExprType::Trunc)),
+                ("abs", raw_call(ExprType::Abs)),
+                ("exp", raw_call(ExprType::Exp)),
+                ("ln", raw_call(ExprType::Ln)),
+                ("log", raw_call(ExprType::Log10)),
+                ("log10", raw_call(ExprType::Log10)),
+                ("mod", raw_call(ExprType::Modulus)),
+                ("sin", raw_call(ExprType::Sin)),
+                ("cos", raw_call(ExprType::Cos)),
+                ("tan", raw_call(ExprType::Tan)),
+                ("cot", raw_call(ExprType::Cot)),
+                ("asin", raw_call(ExprType::Asin)),
+                ("acos", raw_call(ExprType::Acos)),
+                ("atan", raw_call(ExprType::Atan)),
+                ("atan2", raw_call(ExprType::Atan2)),
+                ("sind", raw_call(ExprType::Sind)),
+                ("cosd", raw_call(ExprType::Cosd)),
+                ("cotd", raw_call(ExprType::Cotd)),
+                ("tand", raw_call(ExprType::Tand)),
+                ("sinh", raw_call(ExprType::Sinh)),
+                ("cosh", raw_call(ExprType::Cosh)),
+                ("tanh", raw_call(ExprType::Tanh)),
+                ("coth", raw_call(ExprType::Coth)),
+                ("asinh", raw_call(ExprType::Asinh)),
+                ("acosh", raw_call(ExprType::Acosh)),
+                ("atanh", raw_call(ExprType::Atanh)),
+                ("asind", raw_call(ExprType::Asind)),
+                ("acosd", raw_call(ExprType::Acosd)),
+                ("degrees", raw_call(ExprType::Degrees)),
+                ("radians", raw_call(ExprType::Radians)),
+                ("sqrt", raw_call(ExprType::Sqrt)),
+                ("cbrt", raw_call(ExprType::Cbrt)),
+                ("sign", raw_call(ExprType::Sign)),
+                ("scale", raw_call(ExprType::Scale)),
+                ("min_scale", raw_call(ExprType::MinScale)),
+                ("trim_scale", raw_call(ExprType::TrimScale)),
+
+                (
+                    "to_timestamp",
+                    dispatch_by_len(vec![
+                        (1, raw_call(ExprType::SecToTimestamptz)),
+                        (2, raw_call(ExprType::CharToTimestamptz)),
+                    ]),
+                ),
+                ("date_trunc", raw_call(ExprType::DateTrunc)),
+                ("date_part", raw_call(ExprType::DatePart)),
+                ("make_date", raw_call(ExprType::MakeDate)),
+                ("make_time", raw_call(ExprType::MakeTime)),
+                ("make_timestamp", raw_call(ExprType::MakeTimestamp)),
+                ("to_date", raw_call(ExprType::CharToDate)),
+                ("make_timestamptz", raw_call(ExprType::MakeTimestamptz)),
+                // string
+                ("substr", raw_call(ExprType::Substr)),
+                ("length", raw_call(ExprType::Length)),
+                ("upper", raw_call(ExprType::Upper)),
+                ("lower", raw_call(ExprType::Lower)),
+                ("trim", raw_call(ExprType::Trim)),
+                ("replace", raw_call(ExprType::Replace)),
+                ("overlay", raw_call(ExprType::Overlay)),
+                ("btrim", raw_call(ExprType::Trim)),
+                ("ltrim", raw_call(ExprType::Ltrim)),
+                ("rtrim", raw_call(ExprType::Rtrim)),
+                ("md5", raw_call(ExprType::Md5)),
+                ("to_char", raw_call(ExprType::ToChar)),
+                (
+                    "concat",
+                    rewrite(ExprType::ConcatWs, rewrite_concat_to_concat_ws),
+                ),
+                ("concat_ws", raw_call(ExprType::ConcatWs)),
+                ("format", raw_call(ExprType::Format)),
+                ("translate", raw_call(ExprType::Translate)),
+                ("split_part", raw_call(ExprType::SplitPart)),
+                ("char_length", raw_call(ExprType::CharLength)),
+                ("character_length", raw_call(ExprType::CharLength)),
+                ("repeat", raw_call(ExprType::Repeat)),
+                ("ascii", raw_call(ExprType::Ascii)),
+                ("octet_length", raw_call(ExprType::OctetLength)),
+                ("bit_length", raw_call(ExprType::BitLength)),
+                ("regexp_match", raw_call(ExprType::RegexpMatch)),
+                ("regexp_replace", raw_call(ExprType::RegexpReplace)),
+                ("regexp_count", raw_call(ExprType::RegexpCount)),
+                ("regexp_split_to_array", raw_call(ExprType::RegexpSplitToArray)),
+                ("chr", raw_call(ExprType::Chr)),
+                ("starts_with", raw_call(ExprType::StartsWith)),
+                ("initcap", raw_call(ExprType::Initcap)),
+                ("lpad", raw_call(ExprType::Lpad)),
+                ("rpad", raw_call(ExprType::Rpad)),
+                ("reverse", raw_call(ExprType::Reverse)),
+                ("strpos", raw_call(ExprType::Position)),
+                ("to_ascii", raw_call(ExprType::ToAscii)),
+                ("to_hex", raw_call(ExprType::ToHex)),
+                ("quote_ident", raw_call(ExprType::QuoteIdent)),
+                ("quote_literal", guard_by_len(1, raw(|_binder, mut inputs| {
+                    if inputs[0].return_type() != DataType::Varchar {
+                        // Support `quote_literal(any)` by converting it to `quote_literal(any::text)`
+                        // Ref. https://github.com/postgres/postgres/blob/REL_16_1/src/include/catalog/pg_proc.dat#L4641
+                        FunctionCall::cast_mut(&mut inputs[0], DataType::Varchar, CastContext::Explicit)?;
+                    }
+                    Ok(FunctionCall::new_unchecked(ExprType::QuoteLiteral, inputs, DataType::Varchar).into())
+                }))),
+                ("quote_nullable", guard_by_len(1, raw(|_binder, mut inputs| {
+                    if inputs[0].return_type() != DataType::Varchar {
+                        // Support `quote_nullable(any)` by converting it to `quote_nullable(any::text)`
+                        // Ref. https://github.com/postgres/postgres/blob/REL_16_1/src/include/catalog/pg_proc.dat#L4650
+                        FunctionCall::cast_mut(&mut inputs[0], DataType::Varchar, CastContext::Explicit)?;
+                    }
+                    Ok(FunctionCall::new_unchecked(ExprType::QuoteNullable, inputs, DataType::Varchar).into())
+                }))),
+                ("string_to_array", raw_call(ExprType::StringToArray)),
+                ("encode", raw_call(ExprType::Encode)),
+                ("decode", raw_call(ExprType::Decode)),
+                ("convert_from", raw_call(ExprType::ConvertFrom)),
+                ("convert_to", raw_call(ExprType::ConvertTo)),
+                ("sha1", raw_call(ExprType::Sha1)),
+                ("sha224", raw_call(ExprType::Sha224)),
+                ("sha256", raw_call(ExprType::Sha256)),
+                ("sha384", raw_call(ExprType::Sha384)),
+                ("sha512", raw_call(ExprType::Sha512)),
+                ("encrypt", raw_call(ExprType::Encrypt)),
+                ("decrypt", raw_call(ExprType::Decrypt)),
+                ("left", raw_call(ExprType::Left)),
+                ("right", raw_call(ExprType::Right)),
+                ("inet_aton", raw_call(ExprType::InetAton)),
+                ("inet_ntoa", raw_call(ExprType::InetNtoa)),
+                ("int8send", raw_call(ExprType::PgwireSend)),
+                ("int8recv", guard_by_len(1, raw(|_binder, mut inputs| {
+                    // Similar to `cast` from string, return type is set explicitly rather than inferred.
+                    let hint = if !inputs[0].is_untyped() && inputs[0].return_type() == DataType::Varchar {
+                        " Consider `decode` or cast."
+                    } else {
+                        ""
+                    };
+                    inputs[0].cast_implicit_mut(DataType::Bytea).map_err(|e| {
+                        ErrorCode::BindError(format!("{} in `recv`.{hint}", e.as_report()))
+                    })?;
+                    Ok(FunctionCall::new_unchecked(ExprType::PgwireRecv, inputs, DataType::Int64).into())
+                }))),
+                // array
+                ("array_cat", raw_call(ExprType::ArrayCat)),
+                ("array_append", raw_call(ExprType::ArrayAppend)),
+                ("array_join", raw_call(ExprType::ArrayToString)),
+                ("array_prepend", raw_call(ExprType::ArrayPrepend)),
+                ("array_to_string", raw_call(ExprType::ArrayToString)),
+                ("array_distinct", raw_call(ExprType::ArrayDistinct)),
+                ("array_min", raw_call(ExprType::ArrayMin)),
+                ("array_sort", raw_call(ExprType::ArraySort)),
+                ("array_length", raw_call(ExprType::ArrayLength)),
+                ("cardinality", raw_call(ExprType::Cardinality)),
+                ("array_remove", raw_call(ExprType::ArrayRemove)),
+                ("array_replace", raw_call(ExprType::ArrayReplace)),
+                ("array_max", raw_call(ExprType::ArrayMax)),
+                ("array_sum", raw_call(ExprType::ArraySum)),
+                ("array_position", raw_call(ExprType::ArrayPosition)),
+                ("array_positions", raw_call(ExprType::ArrayPositions)),
+                ("array_contains", raw_call(ExprType::ArrayContains)),
+                ("arraycontains", raw_call(ExprType::ArrayContains)),
+                ("array_contained", raw_call(ExprType::ArrayContained)),
+                ("arraycontained", raw_call(ExprType::ArrayContained)),
+                ("trim_array", raw_call(ExprType::TrimArray)),
+                (
+                    "array_ndims",
+                    guard_by_len(1, raw(|_binder, inputs| {
+                        inputs[0].ensure_array_type()?;
+
+                        let n = inputs[0].return_type().array_ndims()
+                                .try_into().map_err(|_| ErrorCode::BindError("array_ndims integer overflow".into()))?;
+                        Ok(ExprImpl::literal_int(n))
+                    })),
+                ),
+                (
+                    "array_lower",
+                    guard_by_len(2, raw(|binder, inputs| {
+                        let (arg0, arg1) = inputs.into_iter().next_tuple().unwrap();
+                        // rewrite into `CASE WHEN 0 < arg1 AND arg1 <= array_ndims(arg0) THEN 1 END`
+                        let ndims_expr = binder.bind_builtin_scalar_function("array_ndims", vec![arg0], false)?;
+                        let arg1 = arg1.cast_implicit(DataType::Int32)?;
+
+                        FunctionCall::new(
+                            ExprType::Case,
+                            vec![
+                                FunctionCall::new(
+                                    ExprType::And,
+                                    vec![
+                                        FunctionCall::new(ExprType::LessThan, vec![ExprImpl::literal_int(0), arg1.clone()])?.into(),
+                                        FunctionCall::new(ExprType::LessThanOrEqual, vec![arg1, ndims_expr])?.into(),
+                                    ],
+                                )?.into(),
+                                ExprImpl::literal_int(1),
+                            ],
+                        ).map(Into::into)
+                    })),
+                ),
+                ("array_upper", raw_call(ExprType::ArrayLength)), // `lower == 1` implies `upper == length`
+                ("array_dims", raw_call(ExprType::ArrayDims)),
+                // int256
+                ("hex_to_int256", raw_call(ExprType::HexToInt256)),
+                // jsonb
+                ("jsonb_object_field", raw_call(ExprType::JsonbAccess)),
+                ("jsonb_array_element", raw_call(ExprType::JsonbAccess)),
+                ("jsonb_object_field_text", raw_call(ExprType::JsonbAccessStr)),
+                ("jsonb_array_element_text", raw_call(ExprType::JsonbAccessStr)),
+                ("jsonb_extract_path", raw_call(ExprType::JsonbExtractPath)),
+                ("jsonb_extract_path_text", raw_call(ExprType::JsonbExtractPathText)),
+                ("jsonb_typeof", raw_call(ExprType::JsonbTypeof)),
+                ("jsonb_array_length", raw_call(ExprType::JsonbArrayLength)),
+                ("jsonb_concat", raw_call(ExprType::JsonbConcat)),
+                ("jsonb_object", raw_call(ExprType::JsonbObject)),
+                ("jsonb_pretty", raw_call(ExprType::JsonbPretty)),
+                ("jsonb_contains", raw_call(ExprType::JsonbContains)),
+                ("jsonb_contained", raw_call(ExprType::JsonbContained)),
+                ("jsonb_exists", raw_call(ExprType::JsonbExists)),
+                ("jsonb_exists_any", raw_call(ExprType::JsonbExistsAny)),
+                ("jsonb_exists_all", raw_call(ExprType::JsonbExistsAll)),
+                ("jsonb_delete", raw_call(ExprType::Subtract)),
+                ("jsonb_delete_path", raw_call(ExprType::JsonbDeletePath)),
+                ("jsonb_strip_nulls", raw_call(ExprType::JsonbStripNulls)),
+                ("to_jsonb", raw_call(ExprType::ToJsonb)),
+                ("jsonb_build_array", raw_call(ExprType::JsonbBuildArray)),
+                ("jsonb_build_object", raw_call(ExprType::JsonbBuildObject)),
+                ("jsonb_populate_record", raw_call(ExprType::JsonbPopulateRecord)),
+                ("jsonb_path_match", raw_call(ExprType::JsonbPathMatch)),
+                ("jsonb_path_exists", raw_call(ExprType::JsonbPathExists)),
+                ("jsonb_path_query_array", raw_call(ExprType::JsonbPathQueryArray)),
+                ("jsonb_path_query_first", raw_call(ExprType::JsonbPathQueryFirst)),
+                ("jsonb_set", raw_call(ExprType::JsonbSet)),
+                // Functions that return a constant value
+                ("pi", pi()),
+                // greatest and least
+                ("greatest", raw_call(ExprType::Greatest)),
+                ("least", raw_call(ExprType::Least)),
+                // System information operations.
+                (
+                    "pg_typeof",
+                    guard_by_len(1, raw(|_binder, inputs| {
+                        let input = &inputs[0];
+                        let v = match input.is_untyped() {
+                            true => "unknown".into(),
+                            false => input.return_type().to_string(),
+                        };
+                        Ok(ExprImpl::literal_varchar(v))
+                    })),
+                ),
+                ("current_database", guard_by_len(0, raw(|binder, _inputs| {
+                    Ok(ExprImpl::literal_varchar(binder.db_name.clone()))
+                }))),
+                ("current_schema", guard_by_len(0, raw(|binder, _inputs| {
+                    return Ok(binder
+                        .first_valid_schema()
+                        .map(|schema| ExprImpl::literal_varchar(schema.name()))
+                        .unwrap_or_else(|_| ExprImpl::literal_null(DataType::Varchar)));
+                }))),
+                ("current_schemas", raw(|binder, mut inputs| {
+                    let no_match_err = ErrorCode::ExprError(
+                            "No function matches the given name and argument types. You might need to add explicit type casts.".into()
+                        );
+                    if inputs.len() != 1 {
+                        return Err(no_match_err.into());
+                    }
+                    let input = inputs
+                        .pop()
+                        .unwrap()
+                        .enforce_bool_clause("current_schemas")
+                        .map_err(|_| no_match_err)?;
+
+                    let ExprImpl::Literal(literal) = &input else {
+                        bail_not_implemented!("Only boolean literals are supported in `current_schemas`.");
+                    };
+
+                    let Some(bool) = literal.get_data().as_ref().map(|bool| bool.clone().into_bool()) else {
+                        return Ok(ExprImpl::literal_null(DataType::List(Box::new(DataType::Varchar))));
+                    };
+
+                    let paths = if bool {
+                        binder.search_path.path()
+                    } else {
+                        binder.search_path.real_path()
+                    };
+
+                    let mut schema_names = vec![];
+                    for path in paths {
+                        let mut schema_name = path;
+                        if schema_name == USER_NAME_WILD_CARD {
+                            schema_name = &binder.auth_context.user_name;
+                        }
+
+                        if binder
+                            .catalog
+                            .get_schema_by_name(&binder.db_name, schema_name)
+                            .is_ok()
+                        {
+                            schema_names.push(schema_name.as_str());
+                        }
+                    }
+
+                    Ok(ExprImpl::literal_list(
+                        ListValue::from_iter(schema_names),
+                        DataType::Varchar,
+                    ))
+                })),
+                ("session_user", session_user()),
+                ("current_role", current_user()),
+                ("current_user", current_user()),
+                ("user", current_user()),
+                ("pg_get_userbyid", raw_call(ExprType::PgGetUserbyid)),
+                ("pg_get_indexdef", raw_call(ExprType::PgGetIndexdef)),
+                ("pg_get_viewdef", raw_call(ExprType::PgGetViewdef)),
+                ("pg_index_column_has_property", raw_call(ExprType::PgIndexColumnHasProperty)),
+                ("pg_relation_size", raw(|_binder, mut inputs|{
+                    if inputs.is_empty() {
+                        return Err(ErrorCode::ExprError(
+                            "function pg_relation_size() does not exist".into(),
+                        )
+                        .into());
+                    }
+                    inputs[0].cast_to_regclass_mut()?;
+                    Ok(FunctionCall::new(ExprType::PgRelationSize, inputs)?.into())
+                })),
+                ("pg_get_serial_sequence", raw_literal(ExprImpl::literal_null(DataType::Varchar))),
+                ("pg_table_size", guard_by_len(1, raw(|_binder, mut inputs|{
+                    inputs[0].cast_to_regclass_mut()?;
+                    Ok(FunctionCall::new(ExprType::PgRelationSize, inputs)?.into())
+                }))),
+                ("pg_indexes_size", guard_by_len(1, raw(|_binder, mut inputs|{
+                    inputs[0].cast_to_regclass_mut()?;
+                    Ok(FunctionCall::new(ExprType::PgIndexesSize, inputs)?.into())
+                }))),
+                ("pg_get_expr", raw(|_binder, inputs|{
+                    if inputs.len() == 2 || inputs.len() == 3 {
+                        // TODO: implement pg_get_expr rather than just return empty as an workaround.
+                        Ok(ExprImpl::literal_varchar("".into()))
+                    } else {
+                        Err(ErrorCode::ExprError(
+                            "Too many/few arguments for pg_catalog.pg_get_expr()".into(),
+                        )
+                        .into())
+                    }
+                })),
+                ("current_setting", guard_by_len(1, raw(|binder, inputs| {
+                    let input = &inputs[0];
+                    let input = if let ExprImpl::Literal(literal) = input &&
+                        let Some(ScalarImpl::Utf8(input)) = literal.get_data()
+                    {
+                        input
+                    } else {
+                        return Err(ErrorCode::ExprError(
+                            "Only literal is supported in `setting_name`.".into(),
+                        )
+                        .into());
+                    };
+                    let session_config = binder.session_config.read();
+                    Ok(ExprImpl::literal_varchar(session_config.get(input.as_ref())?))
+                }))),
+                ("set_config", guard_by_len(3, raw(|binder, inputs| {
+                    let setting_name = if let ExprImpl::Literal(literal) = &inputs[0] && let Some(ScalarImpl::Utf8(input)) = literal.get_data() {
+                        input
+                    } else {
+                        return Err(ErrorCode::ExprError(
+                            "Only string literal is supported in `setting_name`.".into(),
+                        )
+                        .into());
+                    };
+
+                    let new_value = if let ExprImpl::Literal(literal) = &inputs[1] && let Some(ScalarImpl::Utf8(input)) = literal.get_data() {
+                        input
+                    } else {
+                        return Err(ErrorCode::ExprError(
+                            "Only string literal is supported in `setting_name`.".into(),
+                        )
+                        .into());
+                    };
+
+                    let is_local = if let ExprImpl::Literal(literal) = &inputs[2] && let Some(ScalarImpl::Bool(input)) = literal.get_data() {
+                        input
+                    } else {
+                        return Err(ErrorCode::ExprError(
+                            "Only bool literal is supported in `is_local`.".into(),
+                        )
+                        .into());
+                    };
+
+                    if *is_local {
+                        return Err(ErrorCode::ExprError(
+                            "`is_local = true` is not supported now.".into(),
+                        )
+                        .into());
+                    }
+
+                    let mut session_config = binder.session_config.write();
+
+                    // TODO: report session config changes if necessary.
+                    session_config.set(setting_name, new_value.to_string(), &mut())?;
+
+                    Ok(ExprImpl::literal_varchar(new_value.to_string()))
+                }))),
+                ("format_type", raw_call(ExprType::FormatType)),
+                ("pg_table_is_visible", raw_literal(ExprImpl::literal_bool(true))),
+                ("pg_type_is_visible", raw_literal(ExprImpl::literal_bool(true))),
+                ("pg_get_constraintdef", raw_literal(ExprImpl::literal_null(DataType::Varchar))),
+                ("pg_get_partkeydef", raw_literal(ExprImpl::literal_null(DataType::Varchar))),
+                ("pg_encoding_to_char", raw_literal(ExprImpl::literal_varchar("UTF8".into()))),
+                ("has_database_privilege", raw_literal(ExprImpl::literal_bool(true))),
+                ("has_table_privilege", raw(|binder, mut inputs|{
+                    if inputs.len() == 2 {
+                        inputs.insert(0, ExprImpl::literal_varchar(binder.auth_context.user_name.clone()));
+                    }
+                    if inputs.len() == 3 {
+                        if inputs[1].return_type() == DataType::Varchar {
+                            inputs[1].cast_to_regclass_mut()?;
+                        }
+                        Ok(FunctionCall::new(ExprType::HasTablePrivilege, inputs)?.into())
+                    } else {
+                        Err(ErrorCode::ExprError(
+                            "Too many/few arguments for pg_catalog.has_table_privilege()".into(),
+                        )
+                        .into())
+                    }
+                })),
+                ("has_any_column_privilege", raw(|binder, mut inputs|{
+                    if inputs.len() == 2 {
+                        inputs.insert(0, ExprImpl::literal_varchar(binder.auth_context.user_name.clone()));
+                    }
+                    if inputs.len() == 3 {
+                        if inputs[1].return_type() == DataType::Varchar {
+                            inputs[1].cast_to_regclass_mut()?;
+                        }
+                        Ok(FunctionCall::new(ExprType::HasAnyColumnPrivilege, inputs)?.into())
+                    } else {
+                        Err(ErrorCode::ExprError(
+                            "Too many/few arguments for pg_catalog.has_any_column_privilege()".into(),
+                        )
+                        .into())
+                    }
+                })),
+                ("has_schema_privilege", raw(|binder, mut inputs|{
+                    if inputs.len() == 2 {
+                        inputs.insert(0, ExprImpl::literal_varchar(binder.auth_context.user_name.clone()));
+                    }
+                    if inputs.len() == 3 {
+                        Ok(FunctionCall::new(ExprType::HasSchemaPrivilege, inputs)?.into())
+                    } else {
+                        Err(ErrorCode::ExprError(
+                            "Too many/few arguments for pg_catalog.has_schema_privilege()".into(),
+                        )
+                        .into())
+                    }
+                })),
+                ("pg_stat_get_numscans", raw_literal(ExprImpl::literal_bigint(0))),
+                ("pg_backend_pid", raw(|binder, _inputs| {
+                    // FIXME: the session id is not global unique in multi-frontend env.
+                    Ok(ExprImpl::literal_int(binder.session_id.0))
+                })),
+                ("pg_cancel_backend", guard_by_len(1, raw(|_binder, _inputs| {
+                        // TODO: implement real cancel rather than just return false as an workaround.
+                        Ok(ExprImpl::literal_bool(false))
+                }))),
+                ("pg_terminate_backend", guard_by_len(1, raw(|_binder, _inputs|{
+                        // TODO: implement real terminate rather than just return false as an
+                        // workaround.
+                        Ok(ExprImpl::literal_bool(false))
+                }))),
+                ("pg_tablespace_location", guard_by_len(1, raw_literal(ExprImpl::literal_null(DataType::Varchar)))),
+                ("pg_postmaster_start_time", guard_by_len(0, raw(|_binder, _inputs|{
+                    let server_start_time = risingwave_variables::get_server_start_time();
+                    let datum = server_start_time.map(Timestamptz::from).map(ScalarImpl::from);
+                    let literal = Literal::new(datum, DataType::Timestamptz);
+                    Ok(literal.into())
+                }))),
+                // TODO: really implement them.
+                // https://www.postgresql.org/docs/9.5/functions-info.html#FUNCTIONS-INFO-COMMENT-TABLE
+                // WARN: Hacked in [`Binder::bind_function`]!!!
+                ("col_description", raw_call(ExprType::ColDescription)),
+                ("obj_description", raw_literal(ExprImpl::literal_varchar("".to_string()))),
+                ("shobj_description", raw_literal(ExprImpl::literal_varchar("".to_string()))),
+                ("pg_is_in_recovery", raw_call(ExprType::PgIsInRecovery)),
+                ("rw_recovery_status", raw_call(ExprType::RwRecoveryStatus)),
+                // internal
+                ("rw_vnode", raw_call(ExprType::Vnode)),
+                ("rw_test_paid_tier", raw_call(ExprType::TestPaidTier)), // for testing purposes
+                // TODO: choose which pg version we should return.
+                ("version", raw_literal(ExprImpl::literal_varchar(current_cluster_version()))),
+                // non-deterministic
+                ("now", now()),
+                ("current_timestamp", now()),
+                ("proctime", proctime()),
+                ("pg_sleep", raw_call(ExprType::PgSleep)),
+                ("pg_sleep_for", raw_call(ExprType::PgSleepFor)),
+                // TODO: implement pg_sleep_until
+                // ("pg_sleep_until", raw_call(ExprType::PgSleepUntil)),
+
+                // cast functions
+                // only functions required by the existing PostgreSQL tool are implemented
+                ("date", guard_by_len(1, raw(|_binder, inputs| {
+                    inputs[0].clone().cast_explicit(DataType::Date).map_err(Into::into)
+                }))),
+            ]
+            .into_iter()
+            .collect()
+        });
+
+        static FUNCTIONS_BKTREE: LazyLock<BKTree<&str>> = LazyLock::new(|| {
+            let mut tree = BKTree::new(metrics::Levenshtein);
+
+            // TODO: Also hint other functinos, e.g., Agg or UDF.
+            for k in HANDLES.keys() {
+                tree.add(*k);
+            }
+
+            tree
+        });
+
+        if variadic {
+            let func = match function_name {
+                "format" => ExprType::FormatVariadic,
+                "concat" => ExprType::ConcatVariadic,
+                "concat_ws" => ExprType::ConcatWsVariadic,
+                "jsonb_build_array" => ExprType::JsonbBuildArrayVariadic,
+                "jsonb_build_object" => ExprType::JsonbBuildObjectVariadic,
+                "jsonb_extract_path" => ExprType::JsonbExtractPathVariadic,
+                "jsonb_extract_path_text" => ExprType::JsonbExtractPathTextVariadic,
+                _ => {
+                    return Err(ErrorCode::BindError(format!(
+                        "VARIADIC argument is not allowed in function \"{}\"",
+                        function_name
+                    ))
+                    .into())
+                }
+            };
+            return Ok(FunctionCall::new(func, inputs)?.into());
+        }
+
+        match HANDLES.get(function_name) {
+            Some(handle) => handle(self, inputs),
+            None => {
+                let allowed_distance = if function_name.len() > 3 { 2 } else { 1 };
+
+                let candidates = FUNCTIONS_BKTREE
+                    .find(function_name, allowed_distance)
+                    .map(|(_idx, c)| c)
+                    .join(" or ");
+
+                Err(no_function!(
+                    candidates = (!candidates.is_empty()).then_some(candidates),
+                    "{}({})",
+                    function_name,
+                    inputs.iter().map(|e| e.return_type()).join(", ")
+                )
+                .into())
+            }
+        }
+    }
+
+    fn ensure_now_function_allowed(&self) -> Result<()> {
+        if self.is_for_stream()
+            && !matches!(
+                self.context.clause,
+                Some(Clause::Where)
+                    | Some(Clause::Having)
+                    | Some(Clause::JoinOn)
+                    | Some(Clause::From)
+            )
+        {
+            return Err(ErrorCode::InvalidInputSyntax(format!(
+                "For streaming queries, `NOW()` function is only allowed in `WHERE`, `HAVING`, `ON` and `FROM`. Found in clause: {:?}. \
+                Please please refer to https://www.risingwave.dev/docs/current/sql-pattern-temporal-filters/ for more information",
+                self.context.clause
+            ))
+            .into());
+        }
+        if matches!(self.context.clause, Some(Clause::GeneratedColumn)) {
+            return Err(ErrorCode::InvalidInputSyntax(
+                "Cannot use `NOW()` function in generated columns. Do you want `PROCTIME()`?"
+                    .to_string(),
+            )
+            .into());
+        }
+        Ok(())
+    }
+
+    fn ensure_proctime_function_allowed(&self) -> Result<()> {
+        if !self.is_for_ddl() {
+            return Err(ErrorCode::InvalidInputSyntax(
+                "Function `PROCTIME()` is only allowed in CREATE TABLE/SOURCE. Is `NOW()` what you want?".to_string(),
+            )
+            .into());
+        }
+        Ok(())
+    }
+}
+
+fn rewrite_concat_to_concat_ws(inputs: Vec<ExprImpl>) -> Result<Vec<ExprImpl>> {
+    if inputs.is_empty() {
+        Err(ErrorCode::BindError(
+            "Function `concat` takes at least 1 arguments (0 given)".to_string(),
+        )
+        .into())
+    } else {
+        let inputs = std::iter::once(ExprImpl::literal_varchar("".to_string()))
+            .chain(inputs)
+            .collect();
+        Ok(inputs)
+    }
+}
+
+/// Make sure inputs only have 2 value and rewrite the arguments.
+/// Nullif(expr1,expr2) -> Case(Equal(expr1 = expr2),null,expr1).
+fn rewrite_nullif_to_case_when(inputs: Vec<ExprImpl>) -> Result<Vec<ExprImpl>> {
+    if inputs.len() != 2 {
+        Err(ErrorCode::BindError("Function `nullif` must contain 2 arguments".to_string()).into())
+    } else {
+        let inputs = vec![
+            FunctionCall::new(ExprType::Equal, inputs.clone())?.into(),
+            Literal::new(None, inputs[0].return_type()).into(),
+            inputs[0].clone(),
+        ];
+        Ok(inputs)
+    }
+}
+
+fn rewrite_two_bool_inputs(mut inputs: Vec<ExprImpl>) -> Result<Vec<ExprImpl>> {
+    if inputs.len() != 2 {
+        return Err(
+            ErrorCode::BindError("function must contain only 2 arguments".to_string()).into(),
+        );
+    }
+    let left = inputs.pop().unwrap();
+    let right = inputs.pop().unwrap();
+    Ok(vec![
+        left.cast_implicit(DataType::Boolean)?,
+        right.cast_implicit(DataType::Boolean)?,
+    ])
+}
diff --git a/src/frontend/src/binder/expr/function/mod.rs b/src/frontend/src/binder/expr/function/mod.rs
new file mode 100644
index 0000000000000..9c2b9e1c644e1
--- /dev/null
+++ b/src/frontend/src/binder/expr/function/mod.rs
@@ -0,0 +1,431 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::HashMap;
+use std::str::FromStr;
+use std::sync::Arc;
+
+use itertools::Itertools;
+use risingwave_common::bail_not_implemented;
+use risingwave_common::catalog::{INFORMATION_SCHEMA_SCHEMA_NAME, PG_CATALOG_SCHEMA_NAME};
+use risingwave_common::types::DataType;
+use risingwave_expr::aggregate::AggKind;
+use risingwave_expr::window_function::WindowFuncKind;
+use risingwave_sqlparser::ast::{self, Function, FunctionArg, FunctionArgExpr, Ident};
+use risingwave_sqlparser::parser::ParserError;
+
+use crate::binder::bind_context::Clause;
+use crate::binder::{Binder, UdfContext};
+use crate::error::{ErrorCode, Result, RwError};
+use crate::expr::{
+    Expr, ExprImpl, ExprType, FunctionCallWithLambda, InputRef, TableFunction, TableFunctionType,
+    UserDefinedFunction,
+};
+
+mod aggregate;
+mod builtin_scalar;
+mod window;
+
+// Defines system functions that without args, ref: https://www.postgresql.org/docs/current/functions-info.html
+const SYS_FUNCTION_WITHOUT_ARGS: &[&str] = &[
+    "session_user",
+    "user",
+    "current_user",
+    "current_role",
+    "current_schema",
+    "current_timestamp",
+];
+
+pub(super) fn is_sys_function_without_args(ident: &Ident) -> bool {
+    SYS_FUNCTION_WITHOUT_ARGS
+        .iter()
+        .any(|e| ident.real_value().as_str() == *e && ident.quote_style().is_none())
+}
+
+/// The global max calling depth for the global counter in `udf_context`
+/// To reduce the chance that the current running rw thread
+/// be killed by os, the current allowance depth of calling
+/// stack is set to `16`.
+const SQL_UDF_MAX_CALLING_DEPTH: u32 = 16;
+
+impl Binder {
+    pub(in crate::binder) fn bind_function(&mut self, f: Function) -> Result<ExprImpl> {
+        let function_name = match f.name.0.as_slice() {
+            [name] => name.real_value(),
+            [schema, name] => {
+                let schema_name = schema.real_value();
+                if schema_name == PG_CATALOG_SCHEMA_NAME {
+                    // pg_catalog is always effectively part of the search path, so we can always bind the function.
+                    // Ref: https://www.postgresql.org/docs/current/ddl-schemas.html#DDL-SCHEMAS-CATALOG
+                    name.real_value()
+                } else if schema_name == INFORMATION_SCHEMA_SCHEMA_NAME {
+                    // definition of information_schema: https://github.com/postgres/postgres/blob/e0b2eed047df9045664da6f724cb42c10f8b12f0/src/backend/catalog/information_schema.sql
+                    //
+                    // FIXME: handle schema correctly, so that the functions are hidden if the schema is not in the search path.
+                    let function_name = name.real_value();
+                    if function_name != "_pg_expandarray" {
+                        bail_not_implemented!(
+                            issue = 12422,
+                            "Unsupported function name under schema: {}",
+                            schema_name
+                        );
+                    }
+                    function_name
+                } else {
+                    bail_not_implemented!(
+                        issue = 12422,
+                        "Unsupported function name under schema: {}",
+                        schema_name
+                    );
+                }
+            }
+            _ => bail_not_implemented!(issue = 112, "qualified function {}", f.name),
+        };
+
+        // FIXME: This is a hack to support [Bytebase queries](https://github.com/TennyZhuang/bytebase/blob/4a26f7c62b80e86e58ad2f77063138dc2f420623/backend/plugin/db/pg/sync.go#L549).
+        // Bytebase widely used the pattern like `obj_description(format('%s.%s',
+        // quote_ident(idx.schemaname), quote_ident(idx.indexname))::regclass) AS comment` to
+        // retrieve object comment, however we don't support casting a non-literal expression to
+        // regclass. We just hack the `obj_description` and `col_description` here, to disable it to
+        // bind its arguments.
+        if function_name == "obj_description" || function_name == "col_description" {
+            return Ok(ExprImpl::literal_varchar("".to_string()));
+        }
+        if function_name == "array_transform" {
+            // For type inference, we need to bind the array type first.
+            return self.bind_array_transform(f);
+        }
+
+        let mut inputs: Vec<_> = f
+            .args
+            .iter()
+            .map(|arg| self.bind_function_arg(arg.clone()))
+            .flatten_ok()
+            .try_collect()?;
+
+        // `aggregate:` on a scalar function
+        if f.scalar_as_agg {
+            let mut scalar_inputs = inputs
+                .iter()
+                .enumerate()
+                .map(|(i, expr)| {
+                    InputRef::new(i, DataType::List(Box::new(expr.return_type()))).into()
+                })
+                .collect_vec();
+            let scalar: ExprImpl = if let Ok(schema) = self.first_valid_schema()
+                && let Some(func) =
+                    schema.get_function_by_name_inputs(&function_name, &mut scalar_inputs)
+            {
+                if !func.kind.is_scalar() {
+                    return Err(ErrorCode::InvalidInputSyntax(
+                        "expect a scalar function after `aggregate:`".to_string(),
+                    )
+                    .into());
+                }
+                UserDefinedFunction::new(func.clone(), scalar_inputs).into()
+            } else {
+                self.bind_builtin_scalar_function(&function_name, scalar_inputs, f.variadic)?
+            };
+            return self.bind_aggregate_function(f, AggKind::WrapScalar(scalar.to_expr_proto()));
+        }
+
+        // user defined function
+        // TODO: resolve schema name https://github.com/risingwavelabs/risingwave/issues/12422
+        if let Ok(schema) = self.first_valid_schema()
+            && let Some(func) = schema.get_function_by_name_inputs(&function_name, &mut inputs)
+        {
+            use crate::catalog::function_catalog::FunctionKind::*;
+
+            if func.language == "sql" {
+                if func.body.is_none() {
+                    return Err(ErrorCode::InvalidInputSyntax(
+                        "`body` must exist for sql udf".to_string(),
+                    )
+                    .into());
+                }
+
+                // This represents the current user defined function is `language sql`
+                let parse_result = risingwave_sqlparser::parser::Parser::parse_sql(
+                    func.body.as_ref().unwrap().as_str(),
+                );
+                if let Err(ParserError::ParserError(err)) | Err(ParserError::TokenizerError(err)) =
+                    parse_result
+                {
+                    // Here we just return the original parse error message
+                    return Err(ErrorCode::InvalidInputSyntax(err).into());
+                }
+
+                debug_assert!(parse_result.is_ok());
+
+                // We can safely unwrap here
+                let ast = parse_result.unwrap();
+
+                // Stash the current `udf_context`
+                // Note that the `udf_context` may be empty,
+                // if the current binding is the root (top-most) sql udf.
+                // In this case the empty context will be stashed
+                // and restored later, no need to maintain other flags.
+                let stashed_udf_context = self.udf_context.get_context();
+
+                // The actual inline logic for sql udf
+                // Note that we will always create new udf context for each sql udf
+                let Ok(context) = UdfContext::create_udf_context(&f.args, &Arc::clone(func)) else {
+                    return Err(ErrorCode::InvalidInputSyntax(
+                        "failed to create the `udf_context`, please recheck your function definition and syntax".to_string()
+                    )
+                    .into());
+                };
+
+                let mut udf_context = HashMap::new();
+                for (c, e) in context {
+                    // Note that we need to bind the args before actual delve in the function body
+                    // This will update the context in the subsequent inner calling function
+                    // e.g.,
+                    // - create function print(INT) returns int language sql as 'select $1';
+                    // - create function print_add_one(INT) returns int language sql as 'select print($1 + 1)';
+                    // - select print_add_one(1); # The result should be 2 instead of 1.
+                    // Without the pre-binding here, the ($1 + 1) will not be correctly populated,
+                    // causing the result to always be 1.
+                    let Ok(e) = self.bind_expr(e) else {
+                        return Err(ErrorCode::BindError(
+                            "failed to bind the argument, please recheck the syntax".to_string(),
+                        )
+                        .into());
+                    };
+                    udf_context.insert(c, e);
+                }
+                self.udf_context.update_context(udf_context);
+
+                // Check for potential recursive calling
+                if self.udf_context.global_count() >= SQL_UDF_MAX_CALLING_DEPTH {
+                    return Err(ErrorCode::BindError(format!(
+                        "function {} calling stack depth limit exceeded",
+                        &function_name
+                    ))
+                    .into());
+                } else {
+                    // Update the status for the global counter
+                    self.udf_context.incr_global_count();
+                }
+
+                if let Ok(expr) = UdfContext::extract_udf_expression(ast) {
+                    let bind_result = self.bind_expr(expr);
+
+                    // We should properly decrement global count after a successful binding
+                    // Since the subsequent probe operation in `bind_column` or
+                    // `bind_parameter` relies on global counting
+                    self.udf_context.decr_global_count();
+
+                    // Restore context information for subsequent binding
+                    self.udf_context.update_context(stashed_udf_context);
+
+                    return bind_result;
+                } else {
+                    return Err(ErrorCode::InvalidInputSyntax(
+                        "failed to parse the input query and extract the udf expression,
+                        please recheck the syntax"
+                            .to_string(),
+                    )
+                    .into());
+                }
+            } else {
+                match &func.kind {
+                    Scalar { .. } => {
+                        return Ok(UserDefinedFunction::new(func.clone(), inputs).into())
+                    }
+                    Table { .. } => {
+                        self.ensure_table_function_allowed()?;
+                        return Ok(TableFunction::new_user_defined(func.clone(), inputs).into());
+                    }
+                    Aggregate => {
+                        return self.bind_aggregate_function(
+                            f,
+                            AggKind::UserDefined(func.as_ref().into()),
+                        );
+                    }
+                }
+            }
+        }
+
+        // agg calls
+        if f.over.is_none()
+            && let Ok(kind) = function_name.parse()
+        {
+            return self.bind_aggregate_function(f, AggKind::Builtin(kind));
+        }
+
+        if f.distinct || !f.order_by.is_empty() || f.filter.is_some() {
+            return Err(ErrorCode::InvalidInputSyntax(format!(
+                    "DISTINCT, ORDER BY or FILTER is only allowed in aggregation functions, but `{}` is not an aggregation function", function_name
+                )
+                )
+                .into());
+        }
+
+        // window function
+        let window_func_kind = WindowFuncKind::from_str(function_name.as_str());
+        if let Ok(kind) = window_func_kind {
+            if let Some(window_spec) = f.over {
+                return self.bind_window_function(kind, inputs, window_spec);
+            }
+            return Err(ErrorCode::InvalidInputSyntax(format!(
+                "Window function `{}` must have OVER clause",
+                function_name
+            ))
+            .into());
+        } else if f.over.is_some() {
+            bail_not_implemented!(
+                issue = 8961,
+                "Unrecognized window function: {}",
+                function_name
+            );
+        }
+
+        // file_scan table function
+        if function_name.eq_ignore_ascii_case("file_scan") {
+            self.ensure_table_function_allowed()?;
+            return Ok(TableFunction::new_file_scan(inputs)?.into());
+        }
+        // table function
+        if let Ok(function_type) = TableFunctionType::from_str(function_name.as_str()) {
+            self.ensure_table_function_allowed()?;
+            return Ok(TableFunction::new(function_type, inputs)?.into());
+        }
+
+        self.bind_builtin_scalar_function(function_name.as_str(), inputs, f.variadic)
+    }
+
+    fn bind_array_transform(&mut self, f: Function) -> Result<ExprImpl> {
+        let [array, lambda] = <[FunctionArg; 2]>::try_from(f.args).map_err(|args| -> RwError {
+            ErrorCode::BindError(format!(
+                "`array_transform` expect two inputs `array` and `lambda`, but {} were given",
+                args.len()
+            ))
+            .into()
+        })?;
+
+        let bound_array = self.bind_function_arg(array)?;
+        let [bound_array] = <[ExprImpl; 1]>::try_from(bound_array).map_err(|bound_array| -> RwError {
+            ErrorCode::BindError(format!("The `array` argument for `array_transform` should be bound to one argument, but {} were got", bound_array.len()))
+                .into()
+        })?;
+
+        let inner_ty = match bound_array.return_type() {
+            DataType::List(ty) => *ty,
+            real_type => {
+                return Err(ErrorCode::BindError(format!(
+                "The `array` argument for `array_transform` should be an array, but {} were got",
+                real_type
+            ))
+                .into())
+            }
+        };
+
+        let ast::FunctionArgExpr::Expr(ast::Expr::LambdaFunction {
+            args: lambda_args,
+            body: lambda_body,
+        }) = lambda.get_expr()
+        else {
+            return Err(ErrorCode::BindError(
+                "The `lambda` argument for `array_transform` should be a lambda function"
+                    .to_string(),
+            )
+            .into());
+        };
+
+        let [lambda_arg] = <[Ident; 1]>::try_from(lambda_args).map_err(|args| -> RwError {
+            ErrorCode::BindError(format!(
+                "The `lambda` argument for `array_transform` should be a lambda function with one argument, but {} were given",
+                args.len()
+            ))
+            .into()
+        })?;
+
+        let bound_lambda = self.bind_unary_lambda_function(inner_ty, lambda_arg, *lambda_body)?;
+
+        let lambda_ret_type = bound_lambda.return_type();
+        let transform_ret_type = DataType::List(Box::new(lambda_ret_type));
+
+        Ok(ExprImpl::FunctionCallWithLambda(Box::new(
+            FunctionCallWithLambda::new_unchecked(
+                ExprType::ArrayTransform,
+                vec![bound_array],
+                bound_lambda,
+                transform_ret_type,
+            ),
+        )))
+    }
+
+    fn bind_unary_lambda_function(
+        &mut self,
+        input_ty: DataType,
+        arg: Ident,
+        body: ast::Expr,
+    ) -> Result<ExprImpl> {
+        let lambda_args = HashMap::from([(arg.real_value(), (0usize, input_ty))]);
+        let orig_lambda_args = self.context.lambda_args.replace(lambda_args);
+        let body = self.bind_expr_inner(body)?;
+        self.context.lambda_args = orig_lambda_args;
+
+        Ok(body)
+    }
+
+    fn ensure_table_function_allowed(&self) -> Result<()> {
+        if let Some(clause) = self.context.clause {
+            match clause {
+                Clause::JoinOn
+                | Clause::Where
+                | Clause::Having
+                | Clause::Filter
+                | Clause::Values
+                | Clause::Insert
+                | Clause::GeneratedColumn => {
+                    return Err(ErrorCode::InvalidInputSyntax(format!(
+                        "table functions are not allowed in {}",
+                        clause
+                    ))
+                    .into());
+                }
+                Clause::GroupBy | Clause::From => {}
+            }
+        }
+        Ok(())
+    }
+
+    pub(in crate::binder) fn bind_function_expr_arg(
+        &mut self,
+        arg_expr: FunctionArgExpr,
+    ) -> Result<Vec<ExprImpl>> {
+        match arg_expr {
+            FunctionArgExpr::Expr(expr) => Ok(vec![self.bind_expr_inner(expr)?]),
+            FunctionArgExpr::QualifiedWildcard(_, _)
+            | FunctionArgExpr::ExprQualifiedWildcard(_, _) => Err(ErrorCode::InvalidInputSyntax(
+                format!("unexpected wildcard {}", arg_expr),
+            )
+            .into()),
+            FunctionArgExpr::Wildcard(None) => Ok(vec![]),
+            FunctionArgExpr::Wildcard(Some(_)) => unreachable!(),
+        }
+    }
+
+    pub(in crate::binder) fn bind_function_arg(
+        &mut self,
+        arg: FunctionArg,
+    ) -> Result<Vec<ExprImpl>> {
+        match arg {
+            FunctionArg::Unnamed(expr) => self.bind_function_expr_arg(expr),
+            FunctionArg::Named { .. } => todo!(),
+        }
+    }
+}
diff --git a/src/frontend/src/binder/expr/function/window.rs b/src/frontend/src/binder/expr/function/window.rs
new file mode 100644
index 0000000000000..3124d4717dd82
--- /dev/null
+++ b/src/frontend/src/binder/expr/function/window.rs
@@ -0,0 +1,282 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use itertools::Itertools;
+use risingwave_common::types::{data_types, DataType, ScalarImpl};
+use risingwave_common::{bail_not_implemented, must_match};
+use risingwave_expr::window_function::{
+    Frame, FrameBound, FrameBounds, FrameExclusion, RangeFrameBounds, RangeFrameOffset,
+    RowsFrameBounds, SessionFrameBounds, SessionFrameGap, WindowFuncKind,
+};
+use risingwave_sqlparser::ast::{
+    self, WindowFrameBound, WindowFrameBounds, WindowFrameExclusion, WindowFrameUnits, WindowSpec,
+};
+
+use crate::binder::Clause;
+use crate::error::{ErrorCode, Result};
+use crate::expr::{Expr, ExprImpl, OrderBy, WindowFunction};
+use crate::Binder;
+
+impl Binder {
+    fn ensure_window_function_allowed(&self) -> Result<()> {
+        if let Some(clause) = self.context.clause {
+            match clause {
+                Clause::Where
+                | Clause::Values
+                | Clause::GroupBy
+                | Clause::Having
+                | Clause::Filter
+                | Clause::GeneratedColumn
+                | Clause::From
+                | Clause::Insert
+                | Clause::JoinOn => {
+                    return Err(ErrorCode::InvalidInputSyntax(format!(
+                        "window functions are not allowed in {}",
+                        clause
+                    ))
+                    .into());
+                }
+            }
+        }
+        Ok(())
+    }
+
+    /// Bind window function calls according to PostgreSQL syntax.
+    /// See <https://www.postgresql.org/docs/current/sql-expressions.html#SYNTAX-WINDOW-FUNCTIONS> for syntax detail.
+    pub(super) fn bind_window_function(
+        &mut self,
+        kind: WindowFuncKind,
+        inputs: Vec<ExprImpl>,
+        WindowSpec {
+            partition_by,
+            order_by,
+            window_frame,
+        }: WindowSpec,
+    ) -> Result<ExprImpl> {
+        self.ensure_window_function_allowed()?;
+        let partition_by = partition_by
+            .into_iter()
+            .map(|arg| self.bind_expr_inner(arg))
+            .try_collect()?;
+        let order_by = OrderBy::new(
+            order_by
+                .into_iter()
+                .map(|order_by_expr| self.bind_order_by_expr(order_by_expr))
+                .collect::<Result<_>>()?,
+        );
+        let frame = if let Some(frame) = window_frame {
+            let exclusion = if let Some(exclusion) = frame.exclusion {
+                match exclusion {
+                    WindowFrameExclusion::CurrentRow => FrameExclusion::CurrentRow,
+                    WindowFrameExclusion::Group | WindowFrameExclusion::Ties => {
+                        bail_not_implemented!(
+                            issue = 9124,
+                            "window frame exclusion `{}` is not supported yet",
+                            exclusion
+                        );
+                    }
+                    WindowFrameExclusion::NoOthers => FrameExclusion::NoOthers,
+                }
+            } else {
+                FrameExclusion::NoOthers
+            };
+            let bounds = match frame.units {
+                WindowFrameUnits::Rows => {
+                    let (start, end) = must_match!(frame.bounds, WindowFrameBounds::Bounds { start, end } => (start, end));
+                    let (start, end) = self.bind_window_frame_usize_bounds(start, end)?;
+                    FrameBounds::Rows(RowsFrameBounds { start, end })
+                }
+                unit @ (WindowFrameUnits::Range | WindowFrameUnits::Session) => {
+                    let order_by_expr = order_by
+                        .sort_exprs
+                        .iter()
+                        // for `RANGE | SESSION` frame, there should be exactly one `ORDER BY` column
+                        .exactly_one()
+                        .map_err(|_| {
+                            ErrorCode::InvalidInputSyntax(format!(
+                                "there should be exactly one ordering column for `{}` frame",
+                                unit
+                            ))
+                        })?;
+                    let order_data_type = order_by_expr.expr.return_type();
+                    let order_type = order_by_expr.order_type;
+
+                    let offset_data_type = match &order_data_type {
+                        // for numeric ordering columns, `offset`/`gap` should be the same type
+                        // NOTE: actually in PG it can be a larger type, but we don't support this here
+                        t @ data_types::range_frame_numeric!() => t.clone(),
+                        // for datetime ordering columns, `offset`/`gap` should be interval
+                        t @ data_types::range_frame_datetime!() => {
+                            if matches!(t, DataType::Date | DataType::Time) {
+                                bail_not_implemented!(
+                                    "`{}` frame with offset of type `{}` is not implemented yet, please manually cast the `ORDER BY` column to `timestamp`",
+                                    unit,
+                                    t
+                                );
+                            }
+                            DataType::Interval
+                        }
+                        // other types are not supported
+                        t => {
+                            return Err(ErrorCode::NotSupported(
+                                format!(
+                                    "`{}` frame with offset of type `{}` is not supported",
+                                    unit, t
+                                ),
+                                "Please re-consider the `ORDER BY` column".to_string(),
+                            )
+                            .into())
+                        }
+                    };
+
+                    if unit == WindowFrameUnits::Range {
+                        let (start, end) = must_match!(frame.bounds, WindowFrameBounds::Bounds { start, end } => (start, end));
+                        let (start, end) = self.bind_window_frame_scalar_impl_bounds(
+                            start,
+                            end,
+                            &offset_data_type,
+                        )?;
+                        FrameBounds::Range(RangeFrameBounds {
+                            order_data_type,
+                            order_type,
+                            offset_data_type,
+                            start: start.map(RangeFrameOffset::new),
+                            end: end.map(RangeFrameOffset::new),
+                        })
+                    } else {
+                        let gap = must_match!(frame.bounds, WindowFrameBounds::Gap(gap) => gap);
+                        let gap_value =
+                            self.bind_window_frame_bound_offset(*gap, offset_data_type.clone())?;
+                        FrameBounds::Session(SessionFrameBounds {
+                            order_data_type,
+                            order_type,
+                            gap_data_type: offset_data_type,
+                            gap: SessionFrameGap::new(gap_value),
+                        })
+                    }
+                }
+                WindowFrameUnits::Groups => {
+                    bail_not_implemented!(
+                        issue = 9124,
+                        "window frame in `GROUPS` mode is not supported yet",
+                    );
+                }
+            };
+
+            // Validate the frame bounds, may return `ExprError` to user if the bounds given are not valid.
+            bounds.validate()?;
+
+            Some(Frame { bounds, exclusion })
+        } else {
+            None
+        };
+        Ok(WindowFunction::new(kind, partition_by, order_by, inputs, frame)?.into())
+    }
+
+    fn bind_window_frame_usize_bounds(
+        &mut self,
+        start: WindowFrameBound,
+        end: Option<WindowFrameBound>,
+    ) -> Result<(FrameBound<usize>, FrameBound<usize>)> {
+        let mut convert_offset = |offset: Box<ast::Expr>| -> Result<usize> {
+            let offset = self
+                .bind_window_frame_bound_offset(*offset, DataType::Int64)?
+                .into_int64();
+            if offset < 0 {
+                return Err(ErrorCode::InvalidInputSyntax(
+                    "offset in window frame bounds must be non-negative".to_string(),
+                )
+                .into());
+            }
+            Ok(offset as usize)
+        };
+        let mut convert_bound = |bound| -> Result<FrameBound<usize>> {
+            Ok(match bound {
+                WindowFrameBound::CurrentRow => FrameBound::CurrentRow,
+                WindowFrameBound::Preceding(None) => FrameBound::UnboundedPreceding,
+                WindowFrameBound::Preceding(Some(offset)) => {
+                    FrameBound::Preceding(convert_offset(offset)?)
+                }
+                WindowFrameBound::Following(None) => FrameBound::UnboundedFollowing,
+                WindowFrameBound::Following(Some(offset)) => {
+                    FrameBound::Following(convert_offset(offset)?)
+                }
+            })
+        };
+        let start = convert_bound(start)?;
+        let end = if let Some(end_bound) = end {
+            convert_bound(end_bound)?
+        } else {
+            FrameBound::CurrentRow
+        };
+        Ok((start, end))
+    }
+
+    fn bind_window_frame_scalar_impl_bounds(
+        &mut self,
+        start: WindowFrameBound,
+        end: Option<WindowFrameBound>,
+        offset_data_type: &DataType,
+    ) -> Result<(FrameBound<ScalarImpl>, FrameBound<ScalarImpl>)> {
+        let mut convert_bound = |bound| -> Result<FrameBound<_>> {
+            Ok(match bound {
+                WindowFrameBound::CurrentRow => FrameBound::CurrentRow,
+                WindowFrameBound::Preceding(None) => FrameBound::UnboundedPreceding,
+                WindowFrameBound::Preceding(Some(offset)) => FrameBound::Preceding(
+                    self.bind_window_frame_bound_offset(*offset, offset_data_type.clone())?,
+                ),
+                WindowFrameBound::Following(None) => FrameBound::UnboundedFollowing,
+                WindowFrameBound::Following(Some(offset)) => FrameBound::Following(
+                    self.bind_window_frame_bound_offset(*offset, offset_data_type.clone())?,
+                ),
+            })
+        };
+        let start = convert_bound(start)?;
+        let end = if let Some(end_bound) = end {
+            convert_bound(end_bound)?
+        } else {
+            FrameBound::CurrentRow
+        };
+        Ok((start, end))
+    }
+
+    fn bind_window_frame_bound_offset(
+        &mut self,
+        offset: ast::Expr,
+        cast_to: DataType,
+    ) -> Result<ScalarImpl> {
+        let mut offset = self.bind_expr(offset)?;
+        if !offset.is_const() {
+            return Err(ErrorCode::InvalidInputSyntax(
+                "offset/gap in window frame bounds must be constant".to_string(),
+            )
+            .into());
+        }
+        if offset.cast_implicit_mut(cast_to.clone()).is_err() {
+            return Err(ErrorCode::InvalidInputSyntax(format!(
+                "offset/gap in window frame bounds must be castable to {}",
+                cast_to
+            ))
+            .into());
+        }
+        let offset = offset.fold_const()?;
+        let Some(offset) = offset else {
+            return Err(ErrorCode::InvalidInputSyntax(
+                "offset/gap in window frame bounds must not be NULL".to_string(),
+            )
+            .into());
+        };
+        Ok(offset)
+    }
+}
diff --git a/src/frontend/src/binder/expr/mod.rs b/src/frontend/src/binder/expr/mod.rs
index 363e6f0738fef..308f34a31d447 100644
--- a/src/frontend/src/binder/expr/mod.rs
+++ b/src/frontend/src/binder/expr/mod.rs
@@ -23,7 +23,7 @@ use risingwave_sqlparser::ast::{
     ObjectName, Query, StructField, TrimWhereField, UnaryOperator,
 };
 
-use crate::binder::expr::function::SYS_FUNCTION_WITHOUT_ARGS;
+use crate::binder::expr::function::is_sys_function_without_args;
 use crate::binder::Binder;
 use crate::error::{ErrorCode, Result, RwError};
 use crate::expr::{Expr as _, ExprImpl, ExprType, FunctionCall, InputRef, Parameter, SubqueryKind};
@@ -72,10 +72,7 @@ impl Binder {
             Expr::Row(exprs) => self.bind_row(exprs),
             // input ref
             Expr::Identifier(ident) => {
-                if SYS_FUNCTION_WITHOUT_ARGS
-                    .iter()
-                    .any(|e| ident.real_value().as_str() == *e && ident.quote_style().is_none())
-                {
+                if is_sys_function_without_args(&ident) {
                     // Rewrite a system variable to a function call, e.g. `SELECT current_schema;`
                     // will be rewritten to `SELECT current_schema();`.
                     // NOTE: Here we don't 100% follow the behavior of Postgres, as it doesn't
diff --git a/src/frontend/src/binder/expr/value.rs b/src/frontend/src/binder/expr/value.rs
index a0758a15d4440..e1fc78e884e02 100644
--- a/src/frontend/src/binder/expr/value.rs
+++ b/src/frontend/src/binder/expr/value.rs
@@ -100,7 +100,7 @@ impl Binder {
         Ok(literal)
     }
 
-    fn bind_date_time_field(field: AstDateTimeField) -> DateTimeField {
+    pub(crate) fn bind_date_time_field(field: AstDateTimeField) -> DateTimeField {
         // This is a binder function rather than `impl From<AstDateTimeField> for DateTimeField`,
         // so that the `sqlparser` crate and the `common` crate are kept independent.
         match field {
diff --git a/src/frontend/src/binder/mod.rs b/src/frontend/src/binder/mod.rs
index 8b526a78d53f4..bae186077056b 100644
--- a/src/frontend/src/binder/mod.rs
+++ b/src/frontend/src/binder/mod.rs
@@ -29,6 +29,7 @@ use crate::error::Result;
 mod bind_context;
 mod bind_param;
 mod create;
+mod create_view;
 mod delete;
 mod expr;
 mod for_system;
@@ -43,6 +44,7 @@ mod update;
 mod values;
 
 pub use bind_context::{BindContext, Clause, LateralBindContext};
+pub use create_view::BoundCreateView;
 pub use delete::BoundDelete;
 pub use expr::{bind_data_type, bind_struct_field};
 pub use insert::BoundInsert;
@@ -761,4 +763,267 @@ mod tests {
 
         expected.assert_eq(&format!("{:#?}", bound));
     }
+
+    #[tokio::test]
+    async fn test_bind_approx_percentile() {
+        let stmt = risingwave_sqlparser::parser::Parser::parse_sql(
+            "SELECT approx_percentile(0.5, 0.01) WITHIN GROUP (ORDER BY generate_series) FROM generate_series(1, 100)",
+        ).unwrap().into_iter().next().unwrap();
+        let parse_expected = expect![[r#"
+            Query(
+                Query {
+                    with: None,
+                    body: Select(
+                        Select {
+                            distinct: All,
+                            projection: [
+                                UnnamedExpr(
+                                    Function(
+                                        Function {
+                                            scalar_as_agg: false,
+                                            name: ObjectName(
+                                                [
+                                                    Ident {
+                                                        value: "approx_percentile",
+                                                        quote_style: None,
+                                                    },
+                                                ],
+                                            ),
+                                            args: [
+                                                Unnamed(
+                                                    Expr(
+                                                        Value(
+                                                            Number(
+                                                                "0.5",
+                                                            ),
+                                                        ),
+                                                    ),
+                                                ),
+                                                Unnamed(
+                                                    Expr(
+                                                        Value(
+                                                            Number(
+                                                                "0.01",
+                                                            ),
+                                                        ),
+                                                    ),
+                                                ),
+                                            ],
+                                            variadic: false,
+                                            over: None,
+                                            distinct: false,
+                                            order_by: [],
+                                            filter: None,
+                                            within_group: Some(
+                                                OrderByExpr {
+                                                    expr: Identifier(
+                                                        Ident {
+                                                            value: "generate_series",
+                                                            quote_style: None,
+                                                        },
+                                                    ),
+                                                    asc: None,
+                                                    nulls_first: None,
+                                                },
+                                            ),
+                                        },
+                                    ),
+                                ),
+                            ],
+                            from: [
+                                TableWithJoins {
+                                    relation: TableFunction {
+                                        name: ObjectName(
+                                            [
+                                                Ident {
+                                                    value: "generate_series",
+                                                    quote_style: None,
+                                                },
+                                            ],
+                                        ),
+                                        alias: None,
+                                        args: [
+                                            Unnamed(
+                                                Expr(
+                                                    Value(
+                                                        Number(
+                                                            "1",
+                                                        ),
+                                                    ),
+                                                ),
+                                            ),
+                                            Unnamed(
+                                                Expr(
+                                                    Value(
+                                                        Number(
+                                                            "100",
+                                                        ),
+                                                    ),
+                                                ),
+                                            ),
+                                        ],
+                                        with_ordinality: false,
+                                    },
+                                    joins: [],
+                                },
+                            ],
+                            lateral_views: [],
+                            selection: None,
+                            group_by: [],
+                            having: None,
+                        },
+                    ),
+                    order_by: [],
+                    limit: None,
+                    offset: None,
+                    fetch: None,
+                },
+            )"#]];
+        parse_expected.assert_eq(&format!("{:#?}", stmt));
+
+        let mut binder = mock_binder();
+        let bound = binder.bind(stmt).unwrap();
+
+        let expected = expect![[r#"
+            Query(
+                BoundQuery {
+                    body: Select(
+                        BoundSelect {
+                            distinct: All,
+                            select_items: [
+                                AggCall(
+                                    AggCall {
+                                        agg_kind: Builtin(
+                                            ApproxPercentile,
+                                        ),
+                                        return_type: Float64,
+                                        args: [
+                                            FunctionCall(
+                                                FunctionCall {
+                                                    func_type: Cast,
+                                                    return_type: Float64,
+                                                    inputs: [
+                                                        InputRef(
+                                                            InputRef {
+                                                                index: 0,
+                                                                data_type: Int32,
+                                                            },
+                                                        ),
+                                                    ],
+                                                },
+                                            ),
+                                        ],
+                                        filter: Condition {
+                                            conjunctions: [],
+                                        },
+                                        distinct: false,
+                                        order_by: OrderBy {
+                                            sort_exprs: [
+                                                OrderByExpr {
+                                                    expr: InputRef(
+                                                        InputRef {
+                                                            index: 0,
+                                                            data_type: Int32,
+                                                        },
+                                                    ),
+                                                    order_type: OrderType {
+                                                        direction: Ascending,
+                                                        nulls_are: Largest,
+                                                    },
+                                                },
+                                            ],
+                                        },
+                                        direct_args: [
+                                            Literal {
+                                                data: Some(
+                                                    Float64(
+                                                        OrderedFloat(
+                                                            0.5,
+                                                        ),
+                                                    ),
+                                                ),
+                                                data_type: Some(
+                                                    Float64,
+                                                ),
+                                            },
+                                            Literal {
+                                                data: Some(
+                                                    Float64(
+                                                        OrderedFloat(
+                                                            0.01,
+                                                        ),
+                                                    ),
+                                                ),
+                                                data_type: Some(
+                                                    Float64,
+                                                ),
+                                            },
+                                        ],
+                                    },
+                                ),
+                            ],
+                            aliases: [
+                                Some(
+                                    "approx_percentile",
+                                ),
+                            ],
+                            from: Some(
+                                TableFunction {
+                                    expr: TableFunction(
+                                        FunctionCall {
+                                            function_type: GenerateSeries,
+                                            return_type: Int32,
+                                            args: [
+                                                Literal(
+                                                    Literal {
+                                                        data: Some(
+                                                            Int32(
+                                                                1,
+                                                            ),
+                                                        ),
+                                                        data_type: Some(
+                                                            Int32,
+                                                        ),
+                                                    },
+                                                ),
+                                                Literal(
+                                                    Literal {
+                                                        data: Some(
+                                                            Int32(
+                                                                100,
+                                                            ),
+                                                        ),
+                                                        data_type: Some(
+                                                            Int32,
+                                                        ),
+                                                    },
+                                                ),
+                                            ],
+                                        },
+                                    ),
+                                    with_ordinality: false,
+                                },
+                            ),
+                            where_clause: None,
+                            group_by: GroupKey(
+                                [],
+                            ),
+                            having: None,
+                            schema: Schema {
+                                fields: [
+                                    approx_percentile:Float64,
+                                ],
+                            },
+                        },
+                    ),
+                    order: [],
+                    limit: None,
+                    offset: None,
+                    with_ties: false,
+                    extra_order_exprs: [],
+                },
+            )"#]];
+
+        expected.assert_eq(&format!("{:#?}", bound));
+    }
 }
diff --git a/src/frontend/src/binder/query.rs b/src/frontend/src/binder/query.rs
index 459e1b7921e94..7ad2091e6fb87 100644
--- a/src/frontend/src/binder/query.rs
+++ b/src/frontend/src/binder/query.rs
@@ -46,7 +46,7 @@ pub struct BoundQuery {
 
 impl BoundQuery {
     /// The schema returned by this [`BoundQuery`].
-    pub fn schema(&self) -> &Schema {
+    pub fn schema(&self) -> std::borrow::Cow<'_, Schema> {
         self.body.schema()
     }
 
@@ -295,6 +295,7 @@ impl Binder {
                         SetExpr::SetOperation {
                             op: SetOperator::Union,
                             all,
+                            corresponding,
                             left,
                             right,
                         },
@@ -307,6 +308,12 @@ impl Binder {
                         .into());
                     };
 
+                    // validated in `validate_rcte`
+                    assert!(
+                        !corresponding.is_corresponding(),
+                        "`CORRESPONDING` is not supported in recursive CTE"
+                    );
+
                     let entry = self
                         .context
                         .cte_to_relation
@@ -396,6 +403,7 @@ impl Binder {
         let SetExpr::SetOperation {
             op: SetOperator::Union,
             all,
+            corresponding,
             left,
             right,
         } = body
@@ -412,10 +420,18 @@ impl Binder {
             .into());
         }
 
+        if corresponding.is_corresponding() {
+            return Err(ErrorCode::BindError(
+                "`CORRESPONDING` is not supported in recursive CTE".to_string(),
+            )
+            .into());
+        }
+
         Ok((
             SetExpr::SetOperation {
                 op: SetOperator::Union,
                 all,
+                corresponding,
                 left,
                 right,
             },
@@ -468,7 +484,7 @@ impl Binder {
         self.context.cte_to_relation = new_context.cte_to_relation;
 
         Self::align_schema(&mut base, &mut recursive, SetOperator::Union)?;
-        let schema = base.schema().clone();
+        let schema = base.schema().into_owned();
 
         let recursive_union = RecursiveUnion {
             all,
diff --git a/src/frontend/src/binder/relation/table_function.rs b/src/frontend/src/binder/relation/table_function.rs
index 1a609f87670f3..22b9c2a344c2c 100644
--- a/src/frontend/src/binder/relation/table_function.rs
+++ b/src/frontend/src/binder/relation/table_function.rs
@@ -83,6 +83,7 @@ impl Binder {
         let mut clause = Some(Clause::From);
         std::mem::swap(&mut self.context.clause, &mut clause);
         let func = self.bind_function(Function {
+            scalar_as_agg: false,
             name,
             args,
             variadic: false,
diff --git a/src/frontend/src/binder/set_expr.rs b/src/frontend/src/binder/set_expr.rs
index be4943d59defd..68af5845bf7a4 100644
--- a/src/frontend/src/binder/set_expr.rs
+++ b/src/frontend/src/binder/set_expr.rs
@@ -12,12 +12,17 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::borrow::Cow;
+use std::collections::HashMap;
+
 use risingwave_common::bail_not_implemented;
 use risingwave_common::catalog::Schema;
+use risingwave_common::util::column_index_mapping::ColIndexMapping;
 use risingwave_common::util::iter_util::ZipEqFast;
-use risingwave_sqlparser::ast::{SetExpr, SetOperator};
+use risingwave_sqlparser::ast::{Corresponding, SetExpr, SetOperator};
 
 use super::statement::RewriteExprsRecursive;
+use super::UNNAMED_COLUMN;
 use crate::binder::{BindContext, Binder, BoundQuery, BoundSelect, BoundValues};
 use crate::error::{ErrorCode, Result};
 use crate::expr::{align_types, CorrelatedId, Depth};
@@ -33,6 +38,8 @@ pub enum BoundSetExpr {
     SetOperation {
         op: BoundSetOperation,
         all: bool,
+        // Corresponding columns of the left and right side.
+        corresponding_col_indices: Option<(ColIndexMapping, ColIndexMapping)>,
         left: Box<BoundSetExpr>,
         right: Box<BoundSetExpr>,
     },
@@ -72,12 +79,29 @@ impl From<SetOperator> for BoundSetOperation {
 impl BoundSetExpr {
     /// The schema returned by this [`BoundSetExpr`].
 
-    pub fn schema(&self) -> &Schema {
+    pub fn schema(&self) -> Cow<'_, Schema> {
         match self {
-            BoundSetExpr::Select(s) => s.schema(),
-            BoundSetExpr::Values(v) => v.schema(),
+            BoundSetExpr::Select(s) => Cow::Borrowed(s.schema()),
+            BoundSetExpr::Values(v) => Cow::Borrowed(v.schema()),
             BoundSetExpr::Query(q) => q.schema(),
-            BoundSetExpr::SetOperation { left, .. } => left.schema(),
+            BoundSetExpr::SetOperation {
+                left,
+                corresponding_col_indices,
+                ..
+            } => {
+                if let Some((mapping_l, _)) = corresponding_col_indices {
+                    let mut schema = vec![None; mapping_l.target_size()];
+                    for (src, tar) in mapping_l.mapping_pairs() {
+                        assert_eq!(schema[tar], None);
+                        schema[tar] = Some(left.schema().fields[src].clone());
+                    }
+                    Cow::Owned(Schema::new(
+                        schema.into_iter().map(|x| x.unwrap()).collect(),
+                    ))
+                } else {
+                    left.schema()
+                }
+            }
         }
     }
 
@@ -194,6 +218,92 @@ impl Binder {
         Ok(())
     }
 
+    /// Check the corresponding specification of the set operation.
+    /// Returns the corresponding column index of the left and right side.
+    fn corresponding(
+        &self,
+        left: &BoundSetExpr,
+        right: &BoundSetExpr,
+        corresponding: Corresponding,
+        op: &SetOperator,
+    ) -> Result<(ColIndexMapping, ColIndexMapping)> {
+        let check_duplicate_name = |set_expr: &BoundSetExpr| {
+            let mut name2idx = HashMap::new();
+            for (idx, field) in set_expr.schema().fields.iter().enumerate() {
+                if name2idx.insert(field.name.clone(), idx).is_some() {
+                    return Err(ErrorCode::InvalidInputSyntax(format!(
+                        "Duplicated column name `{}` in a column list of the query in a {} operation. Column list of the query: ({}).",
+                        field.name, op, set_expr.schema().formatted_col_names(),
+                    )));
+                }
+            }
+            Ok(name2idx)
+        };
+
+        // Within the columns of both side, the same <column name> shall not
+        // be specified more than once.
+        let name2idx_l = check_duplicate_name(left)?;
+        let name2idx_r = check_duplicate_name(right)?;
+
+        let mut corresponding_col_idx_l = vec![];
+        let mut corresponding_col_idx_r = vec![];
+
+        if let Some(column_list) = corresponding.column_list() {
+            // The select list of the corresponding set operation should be in the order of <corresponding column list>
+            for column in column_list {
+                let col_name = column.real_value();
+                if let Some(idx_l) = name2idx_l.get(&col_name)
+                    && let Some(idx_r) = name2idx_l.get(&col_name)
+                {
+                    corresponding_col_idx_l.push(*idx_l);
+                    corresponding_col_idx_r.push(*idx_r);
+                } else {
+                    return Err(ErrorCode::InvalidInputSyntax(format!(
+                        "Column name `{}` in CORRESPONDING BY is not found in a side of the {} operation. \
+                        It shall be included in both sides.",
+                        col_name,
+                        op,
+                    )).into());
+                }
+            }
+        } else {
+            // The select list of the corresponding set operation should be
+            // in the order that appears in the <column name>s of the left side.
+            for field in &left.schema().fields {
+                let col_name = &field.name;
+                if col_name != UNNAMED_COLUMN
+                    && let Some(idx_l) = name2idx_l.get(col_name)
+                    && let Some(idx_r) = name2idx_r.get(col_name)
+                {
+                    corresponding_col_idx_l.push(*idx_l);
+                    corresponding_col_idx_r.push(*idx_r);
+                }
+            }
+
+            if corresponding_col_idx_l.is_empty() {
+                return Err(ErrorCode::InvalidInputSyntax(
+                    format!(
+                        "When CORRESPONDING is specified, at least one column of the left side \
+                        shall have a column name that is the column name of some column of the right side in a {} operation. \
+                        Left side query column list: ({}). \
+                        Right side query column list: ({}).",
+                        op,
+                        left.schema().formatted_col_names(),
+                        right.schema().formatted_col_names(),
+                    )
+                )
+                .into());
+            }
+        }
+
+        let corresponding_mapping_l =
+            ColIndexMapping::with_remaining_columns(&corresponding_col_idx_l, left.schema().len());
+        let corresponding_mapping_r =
+            ColIndexMapping::with_remaining_columns(&corresponding_col_idx_r, right.schema().len());
+
+        Ok((corresponding_mapping_l, corresponding_mapping_r))
+    }
+
     pub(super) fn bind_set_expr(&mut self, set_expr: SetExpr) -> Result<BoundSetExpr> {
         match set_expr {
             SetExpr::Select(s) => Ok(BoundSetExpr::Select(Box::new(self.bind_select(*s)?))),
@@ -202,6 +312,7 @@ impl Binder {
             SetExpr::SetOperation {
                 op,
                 all,
+                corresponding,
                 left,
                 right,
             } => {
@@ -215,15 +326,19 @@ impl Binder {
                             .clone_from(&new_context.cte_to_relation);
                         let mut right = self.bind_set_expr(*right)?;
 
-                        if left.schema().fields.len() != right.schema().fields.len() {
-                            return Err(ErrorCode::InvalidInputSyntax(format!(
-                                "each {} query must have the same number of columns",
-                                op
-                            ))
-                            .into());
-                        }
-
-                        Self::align_schema(&mut left, &mut right, op.clone())?;
+                        let corresponding_col_indices = if corresponding.is_corresponding() {
+                            Some(Self::corresponding(
+                                self,
+                                &left,
+                                &right,
+                                corresponding,
+                                &op,
+                            )?)
+                            // TODO: Align schema
+                        } else {
+                            Self::align_schema(&mut left, &mut right, op.clone())?;
+                            None
+                        };
 
                         if all {
                             match op {
@@ -243,6 +358,7 @@ impl Binder {
                         Ok(BoundSetExpr::SetOperation {
                             op: op.into(),
                             all,
+                            corresponding_col_indices,
                             left: Box::new(left),
                             right: Box::new(right),
                         })
diff --git a/src/frontend/src/binder/statement.rs b/src/frontend/src/binder/statement.rs
index 764ebfca8783f..7fca5ff483dfe 100644
--- a/src/frontend/src/binder/statement.rs
+++ b/src/frontend/src/binder/statement.rs
@@ -18,6 +18,7 @@ use risingwave_sqlparser::ast::Statement;
 
 use super::delete::BoundDelete;
 use super::update::BoundUpdate;
+use crate::binder::create_view::BoundCreateView;
 use crate::binder::{Binder, BoundInsert, BoundQuery};
 use crate::error::Result;
 use crate::expr::ExprRewriter;
@@ -28,6 +29,7 @@ pub enum BoundStatement {
     Delete(Box<BoundDelete>),
     Update(Box<BoundUpdate>),
     Query(Box<BoundQuery>),
+    CreateView(Box<BoundCreateView>),
 }
 
 impl BoundStatement {
@@ -46,6 +48,7 @@ impl BoundStatement {
                 .as_ref()
                 .map_or(vec![], |s| s.fields().into()),
             BoundStatement::Query(q) => q.schema().fields().into(),
+            BoundStatement::CreateView(_) => vec![],
         }
     }
 }
@@ -83,6 +86,31 @@ impl Binder {
 
             Statement::Query(q) => Ok(BoundStatement::Query(self.bind_query(*q)?.into())),
 
+            // Note(eric): Can I just bind CreateView to Query??
+            Statement::CreateView {
+                or_replace,
+                materialized,
+                if_not_exists,
+                name,
+                columns,
+                query,
+                emit_mode,
+                with_options,
+            } => {
+                let query = self.bind_query(*query)?;
+                let create_view = BoundCreateView::new(
+                    or_replace,
+                    materialized,
+                    if_not_exists,
+                    name,
+                    columns,
+                    query,
+                    emit_mode,
+                    with_options,
+                );
+                Ok(BoundStatement::CreateView(create_view.into()))
+            }
+
             _ => bail_not_implemented!("unsupported statement {:?}", stmt),
         }
     }
@@ -99,6 +127,7 @@ impl RewriteExprsRecursive for BoundStatement {
             BoundStatement::Delete(inner) => inner.rewrite_exprs_recursive(rewriter),
             BoundStatement::Update(inner) => inner.rewrite_exprs_recursive(rewriter),
             BoundStatement::Query(inner) => inner.rewrite_exprs_recursive(rewriter),
+            BoundStatement::CreateView(inner) => inner.rewrite_exprs_recursive(rewriter),
         }
     }
 }
diff --git a/src/frontend/src/catalog/catalog_service.rs b/src/frontend/src/catalog/catalog_service.rs
index f2bcdd2b62e12..5f42d1e73e5bb 100644
--- a/src/frontend/src/catalog/catalog_service.rs
+++ b/src/frontend/src/catalog/catalog_service.rs
@@ -26,7 +26,7 @@ use risingwave_pb::catalog::{
 use risingwave_pb::ddl_service::alter_owner_request::Object;
 use risingwave_pb::ddl_service::{
     alter_name_request, alter_set_schema_request, create_connection_request, PbReplaceTablePlan,
-    PbTableJobType, ReplaceTablePlan,
+    PbTableJobType, ReplaceTablePlan, TableJobType,
 };
 use risingwave_pb::meta::PbTableParallelism;
 use risingwave_pb::stream_plan::StreamFragmentGraph;
@@ -92,6 +92,7 @@ pub trait CatalogWriter: Send + Sync {
         table: PbTable,
         graph: StreamFragmentGraph,
         mapping: ColIndexMapping,
+        job_type: TableJobType,
     ) -> Result<()>;
 
     async fn alter_source_column(&self, source: PbSource) -> Result<()>;
@@ -212,13 +213,6 @@ pub trait CatalogWriter: Send + Sync {
         object: alter_set_schema_request::Object,
         new_schema_id: u32,
     ) -> Result<()>;
-
-    async fn list_change_log_epochs(
-        &self,
-        table_id: u32,
-        min_epoch: u64,
-        max_count: u32,
-    ) -> Result<Vec<u64>>;
 }
 
 #[derive(Clone)]
@@ -316,10 +310,11 @@ impl CatalogWriter for CatalogWriterImpl {
         table: PbTable,
         graph: StreamFragmentGraph,
         mapping: ColIndexMapping,
+        job_type: TableJobType,
     ) -> Result<()> {
         let version = self
             .meta_client
-            .replace_table(source, table, graph, mapping)
+            .replace_table(source, table, graph, mapping, job_type)
             .await?;
         self.wait_version(version).await
     }
@@ -596,18 +591,6 @@ impl CatalogWriter for CatalogWriterImpl {
 
         Ok(())
     }
-
-    async fn list_change_log_epochs(
-        &self,
-        table_id: u32,
-        min_epoch: u64,
-        max_count: u32,
-    ) -> Result<Vec<u64>> {
-        Ok(self
-            .meta_client
-            .list_change_log_epochs(table_id, min_epoch, max_count)
-            .await?)
-    }
 }
 
 impl CatalogWriterImpl {
diff --git a/src/frontend/src/catalog/function_catalog.rs b/src/frontend/src/catalog/function_catalog.rs
index e042f33f377a8..8782fc10945ab 100644
--- a/src/frontend/src/catalog/function_catalog.rs
+++ b/src/frontend/src/catalog/function_catalog.rs
@@ -87,6 +87,7 @@ impl From<&FunctionCatalog> for PbUserDefinedFunctionMetadata {
         PbUserDefinedFunctionMetadata {
             arg_names: c.arg_names.clone(),
             arg_types: c.arg_types.iter().map(|t| t.to_protobuf()).collect(),
+            return_type: Some(c.return_type.to_protobuf()),
             language: c.language.clone(),
             link: c.link.clone(),
             identifier: c.identifier.clone(),
diff --git a/src/frontend/src/catalog/schema_catalog.rs b/src/frontend/src/catalog/schema_catalog.rs
index 61ec11e144dc4..0394da2a70f81 100644
--- a/src/frontend/src/catalog/schema_catalog.rs
+++ b/src/frontend/src/catalog/schema_catalog.rs
@@ -168,10 +168,7 @@ impl SchemaCatalog {
     pub fn create_index(&mut self, prost: &PbIndex) {
         let name = prost.name.clone();
         let id = prost.id.into();
-
-        let index_table = self
-            .get_created_table_by_id(&prost.index_table_id.into())
-            .unwrap();
+        let index_table = self.get_table_by_id(&prost.index_table_id.into()).unwrap();
         let primary_table = self
             .get_created_table_by_id(&prost.primary_table_id.into())
             .unwrap();
diff --git a/src/frontend/src/catalog/secret_catalog.rs b/src/frontend/src/catalog/secret_catalog.rs
index 5e9aaae7dec99..d1f9048baf0e7 100644
--- a/src/frontend/src/catalog/secret_catalog.rs
+++ b/src/frontend/src/catalog/secret_catalog.rs
@@ -19,7 +19,7 @@ use crate::user::UserId;
 
 #[derive(Clone, Debug, PartialEq, Eq, Hash)]
 pub struct SecretCatalog {
-    pub secret_id: SecretId,
+    pub id: SecretId,
     pub name: String,
     pub database_id: DatabaseId,
     pub value: Vec<u8>,
@@ -29,7 +29,7 @@ pub struct SecretCatalog {
 impl From<&PbSecret> for SecretCatalog {
     fn from(value: &PbSecret) -> Self {
         Self {
-            secret_id: SecretId::new(value.id),
+            id: SecretId::new(value.id),
             database_id: value.database_id,
             owner: value.owner,
             name: value.name.clone(),
diff --git a/src/frontend/src/catalog/source_catalog.rs b/src/frontend/src/catalog/source_catalog.rs
index 1ee095a918e5d..8e64a6db4e2b9 100644
--- a/src/frontend/src/catalog/source_catalog.rs
+++ b/src/frontend/src/catalog/source_catalog.rs
@@ -12,11 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::BTreeMap;
-
 use risingwave_common::catalog::{ColumnCatalog, SourceVersionId};
 use risingwave_common::util::epoch::Epoch;
-use risingwave_connector::WithPropertiesExt;
+use risingwave_connector::{WithOptionsSecResolved, WithPropertiesExt};
 use risingwave_pb::catalog::source::OptionalAssociatedTableId;
 use risingwave_pb::catalog::{PbSource, StreamSourceInfo, WatermarkDesc};
 
@@ -36,7 +34,7 @@ pub struct SourceCatalog {
     pub owner: UserId,
     pub info: StreamSourceInfo,
     pub row_id_index: Option<usize>,
-    pub with_properties: BTreeMap<String, String>,
+    pub with_properties: WithOptionsSecResolved,
     pub watermark_descs: Vec<WatermarkDesc>,
     pub associated_table_id: Option<TableId>,
     pub definition: String,
@@ -46,6 +44,7 @@ pub struct SourceCatalog {
     pub version: SourceVersionId,
     pub created_at_cluster_version: Option<String>,
     pub initialized_at_cluster_version: Option<String>,
+    pub rate_limit: Option<u32>,
 }
 
 impl SourceCatalog {
@@ -55,6 +54,7 @@ impl SourceCatalog {
     }
 
     pub fn to_prost(&self, schema_id: SchemaId, database_id: DatabaseId) -> PbSource {
+        let (with_properties, secret_refs) = self.with_properties.clone().into_parts();
         PbSource {
             id: self.id,
             schema_id,
@@ -63,7 +63,7 @@ impl SourceCatalog {
             row_id_index: self.row_id_index.map(|idx| idx as _),
             columns: self.columns.iter().map(|c| c.to_protobuf()).collect(),
             pk_column_ids: self.pk_col_ids.iter().map(Into::into).collect(),
-            with_properties: self.with_properties.clone().into_iter().collect(),
+            with_properties,
             owner: self.owner,
             info: Some(self.info.clone()),
             watermark_descs: self.watermark_descs.clone(),
@@ -77,7 +77,8 @@ impl SourceCatalog {
             version: self.version,
             created_at_cluster_version: self.created_at_cluster_version.clone(),
             initialized_at_cluster_version: self.initialized_at_cluster_version.clone(),
-            secret_refs: Default::default(),
+            secret_refs,
+            rate_limit: self.rate_limit,
         }
     }
 
@@ -104,7 +105,8 @@ impl From<&PbSource> for SourceCatalog {
             .into_iter()
             .map(Into::into)
             .collect();
-        let with_properties = prost.with_properties.clone().into_iter().collect();
+        let options_with_secrets =
+            WithOptionsSecResolved::new(prost.with_properties.clone(), prost.secret_refs.clone());
         let columns = prost_columns.into_iter().map(ColumnCatalog::from).collect();
         let row_id_index = prost.row_id_index.map(|idx| idx as _);
 
@@ -121,6 +123,7 @@ impl From<&PbSource> for SourceCatalog {
         let version = prost.version;
 
         let connection_id = prost.connection_id;
+        let rate_limit = prost.rate_limit;
 
         Self {
             id,
@@ -131,7 +134,7 @@ impl From<&PbSource> for SourceCatalog {
             owner,
             info: prost.info.clone().unwrap(),
             row_id_index,
-            with_properties,
+            with_properties: options_with_secrets,
             watermark_descs,
             associated_table_id: associated_table_id.map(|x| x.into()),
             definition: prost.definition.clone(),
@@ -141,6 +144,7 @@ impl From<&PbSource> for SourceCatalog {
             version,
             created_at_cluster_version: prost.created_at_cluster_version.clone(),
             initialized_at_cluster_version: prost.initialized_at_cluster_version.clone(),
+            rate_limit,
         }
     }
 }
diff --git a/src/frontend/src/catalog/system_catalog/mod.rs b/src/frontend/src/catalog/system_catalog/mod.rs
index a3b16216d8187..5e32dcb7b2aba 100644
--- a/src/frontend/src/catalog/system_catalog/mod.rs
+++ b/src/frontend/src/catalog/system_catalog/mod.rs
@@ -22,7 +22,6 @@ use std::sync::{Arc, LazyLock};
 use futures::stream::BoxStream;
 use itertools::Itertools;
 use parking_lot::RwLock;
-use risingwave_batch::worker_manager::worker_node_manager::WorkerNodeManagerRef;
 use risingwave_common::acl::AclMode;
 use risingwave_common::array::DataChunk;
 use risingwave_common::catalog::{
@@ -102,8 +101,6 @@ pub struct SysCatalogReaderImpl {
     catalog_reader: CatalogReader,
     // Read user info.
     user_info_reader: UserInfoReader,
-    // Read cluster info.
-    worker_node_manager: WorkerNodeManagerRef,
     // Read from meta.
     meta_client: Arc<dyn FrontendMetaClient>,
     // Read auth context.
@@ -118,7 +115,6 @@ impl SysCatalogReaderImpl {
     pub fn new(
         catalog_reader: CatalogReader,
         user_info_reader: UserInfoReader,
-        worker_node_manager: WorkerNodeManagerRef,
         meta_client: Arc<dyn FrontendMetaClient>,
         auth_context: Arc<AuthContext>,
         config: Arc<RwLock<SessionConfig>>,
@@ -127,7 +123,6 @@ impl SysCatalogReaderImpl {
         Self {
             catalog_reader,
             user_info_reader,
-            worker_node_manager,
             meta_client,
             auth_context,
             config,
diff --git a/src/frontend/src/catalog/system_catalog/pg_catalog/pg_cast.rs b/src/frontend/src/catalog/system_catalog/pg_catalog/pg_cast.rs
index 11bcabcde0f69..d5b1332c25b3e 100644
--- a/src/frontend/src/catalog/system_catalog/pg_catalog/pg_cast.rs
+++ b/src/frontend/src/catalog/system_catalog/pg_catalog/pg_cast.rs
@@ -38,8 +38,8 @@ fn read_pg_cast(_: &SysCatalogReaderImpl) -> Vec<PgCast> {
         .enumerate()
         .map(|(idx, (src, target, ctx))| PgCast {
             oid: idx as i32,
-            castsource: DataType::from(*src).to_oid(),
-            casttarget: DataType::from(*target).to_oid(),
+            castsource: DataType::try_from(*src).unwrap().to_oid(),
+            casttarget: DataType::try_from(*target).unwrap().to_oid(),
             castcontext: ctx.to_string(),
         })
         .collect()
diff --git a/src/frontend/src/catalog/system_catalog/pg_catalog/pg_proc.rs b/src/frontend/src/catalog/system_catalog/pg_catalog/pg_proc.rs
index 259f42376abe9..2f2abce7a0b8e 100644
--- a/src/frontend/src/catalog/system_catalog/pg_catalog/pg_proc.rs
+++ b/src/frontend/src/catalog/system_catalog/pg_catalog/pg_proc.rs
@@ -30,4 +30,5 @@ struct PgProc {
     proargdefaults: i32,
     // Data type of the return value, refer to pg_type.
     prorettype: i32,
+    prokind: String,
 }
diff --git a/src/frontend/src/catalog/system_catalog/rw_catalog/mod.rs b/src/frontend/src/catalog/system_catalog/rw_catalog/mod.rs
index 4e0e165a6d524..879e375e2b762 100644
--- a/src/frontend/src/catalog/system_catalog/rw_catalog/mod.rs
+++ b/src/frontend/src/catalog/system_catalog/rw_catalog/mod.rs
@@ -39,10 +39,10 @@ mod rw_indexes;
 mod rw_internal_tables;
 mod rw_materialized_views;
 mod rw_meta_snapshot;
-mod rw_parallel_units;
 mod rw_relation_info;
 mod rw_relations;
 mod rw_schemas;
+mod rw_secrets;
 mod rw_sinks;
 mod rw_sources;
 mod rw_streaming_parallelism;
diff --git a/src/frontend/src/catalog/system_catalog/rw_catalog/rw_actors.rs b/src/frontend/src/catalog/system_catalog/rw_catalog/rw_actors.rs
index 9769c7cafbf6c..b915a1f9dde97 100644
--- a/src/frontend/src/catalog/system_catalog/rw_catalog/rw_actors.rs
+++ b/src/frontend/src/catalog/system_catalog/rw_catalog/rw_actors.rs
@@ -23,7 +23,7 @@ struct RwActor {
     #[primary_key]
     actor_id: i32,
     fragment_id: i32,
-    parallel_unit_id: i32,
+    worker_id: i32,
     state: String,
 }
 
@@ -36,7 +36,7 @@ async fn read_rw_actors(reader: &SysCatalogReaderImpl) -> Result<Vec<RwActor>> {
         .map(|state| RwActor {
             actor_id: state.actor_id as i32,
             fragment_id: state.fragment_id as i32,
-            parallel_unit_id: state.parallel_unit_id as i32,
+            worker_id: state.worker_id as i32,
             state: state.state().as_str_name().into(),
         })
         .collect())
diff --git a/src/frontend/src/catalog/system_catalog/rw_catalog/rw_hummock_version.rs b/src/frontend/src/catalog/system_catalog/rw_catalog/rw_hummock_version.rs
index f7d265485f706..d4281434c962f 100644
--- a/src/frontend/src/catalog/system_catalog/rw_catalog/rw_hummock_version.rs
+++ b/src/frontend/src/catalog/system_catalog/rw_catalog/rw_hummock_version.rs
@@ -86,13 +86,9 @@ async fn read_hummock_sstables(reader: &SysCatalogReaderImpl) -> Result<Vec<RwHu
 fn remove_key_range_from_version(mut version: HummockVersion) -> HummockVersion {
     // Because key range is too verbose for manual analysis, just don't expose it.
     for cg in version.levels.values_mut() {
-        for level in cg
-            .levels
-            .iter_mut()
-            .chain(cg.l0.as_mut().unwrap().sub_levels.iter_mut())
-        {
+        for level in cg.levels.iter_mut().chain(cg.l0.sub_levels.iter_mut()) {
             for sst in &mut level.table_infos {
-                sst.key_range.take();
+                sst.remove_key_range();
             }
         }
     }
@@ -107,7 +103,7 @@ fn version_to_compaction_group_rows(version: &HummockVersion) -> Vec<RwHummockVe
             version_id: version.id as _,
             max_committed_epoch: version.max_committed_epoch as _,
             safe_epoch: version.visible_table_safe_epoch() as _,
-            compaction_group: json!(cg).into(),
+            compaction_group: json!(cg.to_protobuf()).into(),
         })
         .collect()
 }
@@ -115,9 +111,9 @@ fn version_to_compaction_group_rows(version: &HummockVersion) -> Vec<RwHummockVe
 fn version_to_sstable_rows(version: HummockVersion) -> Vec<RwHummockSstable> {
     let mut sstables = vec![];
     for cg in version.levels.into_values() {
-        for level in cg.levels.into_iter().chain(cg.l0.unwrap().sub_levels) {
+        for level in cg.levels.into_iter().chain(cg.l0.sub_levels) {
             for sst in level.table_infos {
-                let key_range = sst.key_range.unwrap();
+                let key_range = sst.key_range;
                 sstables.push(RwHummockSstable {
                     sstable_id: sst.sst_id as _,
                     object_id: sst.object_id as _,
@@ -125,8 +121,8 @@ fn version_to_sstable_rows(version: HummockVersion) -> Vec<RwHummockSstable> {
                     level_id: level.level_idx as _,
                     sub_level_id: (level.level_idx == 0).then_some(level.sub_level_id as _),
                     level_type: level.level_type as _,
-                    key_range_left: key_range.left,
-                    key_range_right: key_range.right,
+                    key_range_left: key_range.left.to_vec(),
+                    key_range_right: key_range.right.to_vec(),
                     right_exclusive: key_range.right_exclusive,
                     file_size: sst.file_size as _,
                     meta_offset: sst.meta_offset as _,
diff --git a/src/frontend/src/catalog/system_catalog/rw_catalog/rw_hummock_version_deltas.rs b/src/frontend/src/catalog/system_catalog/rw_catalog/rw_hummock_version_deltas.rs
index a46312b817c8d..20e1111e9c799 100644
--- a/src/frontend/src/catalog/system_catalog/rw_catalog/rw_hummock_version_deltas.rs
+++ b/src/frontend/src/catalog/system_catalog/rw_catalog/rw_hummock_version_deltas.rs
@@ -12,8 +12,11 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::collections::HashMap;
+
 use risingwave_common::types::{Fields, JsonbVal};
 use risingwave_frontend_macro::system_catalog;
+use risingwave_pb::hummock::hummock_version_delta::PbGroupDeltas;
 use serde_json::json;
 
 use crate::catalog::system_catalog::SysCatalogReaderImpl;
@@ -39,9 +42,14 @@ async fn read(reader: &SysCatalogReaderImpl) -> Result<Vec<RwHummockVersionDelta
             id: d.id as _,
             prev_id: d.prev_id as _,
             max_committed_epoch: d.max_committed_epoch as _,
-            safe_epoch: d.safe_epoch as _,
+            safe_epoch: d.visible_table_safe_epoch() as _,
             trivial_move: d.trivial_move,
-            group_deltas: json!(d.group_deltas).into(),
+            group_deltas: json!(d
+                .group_deltas
+                .into_iter()
+                .map(|(group_id, deltas)| (group_id, PbGroupDeltas::from(deltas)))
+                .collect::<HashMap<u64, PbGroupDeltas>>())
+            .into(),
         })
         .collect();
     Ok(rows)
diff --git a/src/frontend/src/catalog/system_catalog/rw_catalog/rw_iceberg_files.rs b/src/frontend/src/catalog/system_catalog/rw_catalog/rw_iceberg_files.rs
index 2bb7648920895..b025723857b1e 100644
--- a/src/frontend/src/catalog/system_catalog/rw_catalog/rw_iceberg_files.rs
+++ b/src/frontend/src/catalog/system_catalog/rw_catalog/rw_iceberg_files.rs
@@ -15,7 +15,9 @@
 use std::ops::Deref;
 
 use anyhow::anyhow;
-use icelake::Table;
+use futures::StreamExt;
+use iceberg::spec::ManifestList;
+use iceberg::table::Table;
 use risingwave_common::types::Fields;
 use risingwave_connector::sink::iceberg::IcebergConfig;
 use risingwave_connector::source::ConnectorProperties;
@@ -46,7 +48,7 @@ struct RwIcebergFiles {
     /// Required when content is `EqualityDeletes` and should be null
     /// otherwise. Fields with ids listed in this column must be present
     /// in the delete file
-    pub equality_ids: Option<Vec<i32>>,
+    pub equality_ids: Vec<i32>,
     /// ID representing sort order for this file.
     ///
     /// If sort order ID is missing or unknown, then the order is assumed to
@@ -77,30 +79,40 @@ async fn read(reader: &SysCatalogReaderImpl) -> Result<Vec<RwIcebergFiles>> {
 
     let mut result = vec![];
     for (schema_name, source) in iceberg_sources {
-        let source_props = source.with_properties.clone();
-        let config = ConnectorProperties::extract(source_props, false)?;
+        let config = ConnectorProperties::extract(source.with_properties.clone(), false)?;
         if let ConnectorProperties::Iceberg(iceberg_properties) = config {
             let iceberg_config: IcebergConfig = iceberg_properties.to_iceberg_config();
-            let table: Table = iceberg_config.load_table().await?;
-            result.extend(
-                table
-                    .current_data_files()
+            let table: Table = iceberg_config.load_table_v2().await?;
+            if let Some(snapshot) = table.metadata().current_snapshot() {
+                let manifest_list: ManifestList = snapshot
+                    .load_manifest_list(table.file_io(), table.metadata())
                     .await
-                    .map_err(|e| anyhow!(e))?
-                    .iter()
-                    .map(|file| RwIcebergFiles {
-                        source_id: source.id as i32,
-                        schema_name: schema_name.clone(),
-                        source_name: source.name.clone(),
-                        content: file.content as i32,
-                        file_path: file.file_path.clone(),
-                        file_format: file.file_format.to_string(),
-                        record_count: file.record_count,
-                        file_size_in_bytes: file.file_size_in_bytes,
-                        equality_ids: file.equality_ids.clone(),
-                        sort_order_id: file.sort_order_id,
-                    }),
-            );
+                    .map_err(|e| anyhow!(e))?;
+                for entry in manifest_list.entries() {
+                    let manifest = entry
+                        .load_manifest(table.file_io())
+                        .await
+                        .map_err(|e| anyhow!(e))?;
+                    let mut manifest_entries_stream =
+                        futures::stream::iter(manifest.entries().iter().filter(|e| e.is_alive()));
+
+                    while let Some(manifest_entry) = manifest_entries_stream.next().await {
+                        let file = manifest_entry.data_file();
+                        result.push(RwIcebergFiles {
+                            source_id: source.id as i32,
+                            schema_name: schema_name.clone(),
+                            source_name: source.name.clone(),
+                            content: file.content_type() as i32,
+                            file_path: file.file_path().to_string(),
+                            file_format: file.file_format().to_string(),
+                            record_count: file.record_count() as i64,
+                            file_size_in_bytes: file.file_size_in_bytes() as i64,
+                            equality_ids: file.equality_ids().to_vec(),
+                            sort_order_id: file.sort_order_id(),
+                        });
+                    }
+                }
+            }
         } else {
             unreachable!()
         }
diff --git a/src/frontend/src/catalog/system_catalog/rw_catalog/rw_iceberg_snapshots.rs b/src/frontend/src/catalog/system_catalog/rw_catalog/rw_iceberg_snapshots.rs
index 6e32f48eec844..e2bbcb486b926 100644
--- a/src/frontend/src/catalog/system_catalog/rw_catalog/rw_iceberg_snapshots.rs
+++ b/src/frontend/src/catalog/system_catalog/rw_catalog/rw_iceberg_snapshots.rs
@@ -14,7 +14,7 @@
 
 use std::ops::Deref;
 
-use icelake::Table;
+use iceberg::table::Table;
 use jsonbb::{Value, ValueRef};
 use risingwave_common::types::{Fields, JsonbVal, Timestamptz};
 use risingwave_connector::sink::iceberg::IcebergConfig;
@@ -57,31 +57,30 @@ async fn read(reader: &SysCatalogReaderImpl) -> Result<Vec<RwIcebergSnapshots>>
 
     let mut result = vec![];
     for (schema_name, source) in iceberg_sources {
-        let source_props = source.with_properties.clone();
-        let config = ConnectorProperties::extract(source_props, false)?;
+        let config = ConnectorProperties::extract(source.with_properties.clone(), false)?;
         if let ConnectorProperties::Iceberg(iceberg_properties) = config {
             let iceberg_config: IcebergConfig = iceberg_properties.to_iceberg_config();
-            let table: Table = iceberg_config.load_table().await?;
-            if let Some(snapshots) = &table.current_table_metadata().snapshots {
-                result.extend(snapshots.iter().map(|snapshot| {
-                    RwIcebergSnapshots {
-                        source_id: source.id as i32,
-                        schema_name: schema_name.clone(),
-                        source_name: source.name.clone(),
-                        sequence_number: snapshot.sequence_number,
-                        snapshot_id: snapshot.snapshot_id,
-                        timestamp_ms: Timestamptz::from_millis(snapshot.timestamp_ms),
-                        manifest_list: snapshot.manifest_list.clone(),
-                        summary: Value::object(
-                            snapshot
-                                .summary
-                                .iter()
-                                .map(|(k, v)| (k.as_str(), ValueRef::String(v))),
-                        )
-                        .into(),
-                    }
-                }));
-            }
+            let table: Table = iceberg_config.load_table_v2().await?;
+
+            result.extend(table.metadata().snapshots().map(|snapshot| {
+                RwIcebergSnapshots {
+                    source_id: source.id as i32,
+                    schema_name: schema_name.clone(),
+                    source_name: source.name.clone(),
+                    sequence_number: snapshot.sequence_number(),
+                    snapshot_id: snapshot.snapshot_id(),
+                    timestamp_ms: Timestamptz::from_millis(snapshot.timestamp().timestamp_millis()),
+                    manifest_list: snapshot.manifest_list().to_string(),
+                    summary: Value::object(
+                        snapshot
+                            .summary()
+                            .other
+                            .iter()
+                            .map(|(k, v)| (k.as_str(), ValueRef::String(v))),
+                    )
+                    .into(),
+                }
+            }));
         }
     }
     Ok(result)
diff --git a/src/frontend/src/catalog/system_catalog/rw_catalog/rw_materialized_views.rs b/src/frontend/src/catalog/system_catalog/rw_catalog/rw_materialized_views.rs
index 95b469debc2a0..c593b35b18f87 100644
--- a/src/frontend/src/catalog/system_catalog/rw_catalog/rw_materialized_views.rs
+++ b/src/frontend/src/catalog/system_catalog/rw_catalog/rw_materialized_views.rs
@@ -27,6 +27,7 @@ struct RwMaterializedView {
     schema_id: i32,
     owner: i32,
     definition: String,
+    append_only: bool,
     acl: String,
     initialized_at: Option<Timestamptz>,
     created_at: Option<Timestamptz>,
@@ -50,6 +51,7 @@ fn read_rw_materialized_views(reader: &SysCatalogReaderImpl) -> Result<Vec<RwMat
                 schema_id: schema.id() as i32,
                 owner: table.owner as i32,
                 definition: table.create_sql(),
+                append_only: table.append_only,
                 acl: get_acl_items(
                     &Object::TableId(table.id.table_id),
                     true,
diff --git a/src/frontend/src/catalog/system_catalog/rw_catalog/rw_parallel_units.rs b/src/frontend/src/catalog/system_catalog/rw_catalog/rw_secrets.rs
similarity index 57%
rename from src/frontend/src/catalog/system_catalog/rw_catalog/rw_parallel_units.rs
rename to src/frontend/src/catalog/system_catalog/rw_catalog/rw_secrets.rs
index 0a46e10ec1754..09c9a98798e06 100644
--- a/src/frontend/src/catalog/system_catalog/rw_catalog/rw_parallel_units.rs
+++ b/src/frontend/src/catalog/system_catalog/rw_catalog/rw_secrets.rs
@@ -19,26 +19,27 @@ use crate::catalog::system_catalog::SysCatalogReaderImpl;
 use crate::error::Result;
 
 #[derive(Fields)]
-struct RwParallelUnit {
+struct RwSecret {
     #[primary_key]
     id: i32,
-    worker_id: i32,
+    name: String,
+    owner: i32,
+    acl: String,
 }
 
-#[system_catalog(table, "rw_catalog.rw_parallel_units")]
-fn read_rw_parallel_units(reader: &SysCatalogReaderImpl) -> Result<Vec<RwParallelUnit>> {
-    let workers = reader.worker_node_manager.list_worker_nodes();
+#[system_catalog(table, "rw_catalog.rw_secrets")]
+fn read_rw_view_info(reader: &SysCatalogReaderImpl) -> Result<Vec<RwSecret>> {
+    let catalog_reader = reader.catalog_reader.read_guard();
+    let schemas = catalog_reader.iter_schemas(&reader.auth_context.database)?;
 
-    Ok(workers
-        .into_iter()
-        .flat_map(|worker| {
-            worker
-                .parallel_units
-                .into_iter()
-                .map(move |unit| RwParallelUnit {
-                    id: unit.id as i32,
-                    worker_id: worker.id as i32,
-                })
+    Ok(schemas
+        .flat_map(|schema| {
+            schema.iter_secret().map(|secret| RwSecret {
+                id: secret.id.secret_id() as i32,
+                name: secret.name.clone(),
+                owner: secret.owner as i32,
+                acl: "".into(),
+            })
         })
         .collect())
 }
diff --git a/src/frontend/src/catalog/system_catalog/rw_catalog/rw_tables.rs b/src/frontend/src/catalog/system_catalog/rw_catalog/rw_tables.rs
index d86690eddfe16..78416c97b71af 100644
--- a/src/frontend/src/catalog/system_catalog/rw_catalog/rw_tables.rs
+++ b/src/frontend/src/catalog/system_catalog/rw_catalog/rw_tables.rs
@@ -27,6 +27,7 @@ struct RwTable {
     schema_id: i32,
     owner: i32,
     definition: String,
+    append_only: bool,
     acl: String,
     initialized_at: Option<Timestamptz>,
     created_at: Option<Timestamptz>,
@@ -50,6 +51,7 @@ fn read_rw_table_info(reader: &SysCatalogReaderImpl) -> Result<Vec<RwTable>> {
                 schema_id: schema.id() as i32,
                 owner: table.owner as i32,
                 definition: table.create_sql(),
+                append_only: table.append_only,
                 acl: get_acl_items(
                     &Object::TableId(table.id.table_id),
                     false,
diff --git a/src/frontend/src/catalog/system_catalog/rw_catalog/rw_worker_nodes.rs b/src/frontend/src/catalog/system_catalog/rw_catalog/rw_worker_nodes.rs
index 226b0230e3f21..6c21f524e684e 100644
--- a/src/frontend/src/catalog/system_catalog/rw_catalog/rw_worker_nodes.rs
+++ b/src/frontend/src/catalog/system_catalog/rw_catalog/rw_worker_nodes.rs
@@ -55,7 +55,7 @@ async fn read_rw_worker_nodes_info(reader: &SysCatalogReaderImpl) -> Result<Vec<
                 port: host.map(|h| h.port.to_string()),
                 r#type: worker.get_type().unwrap().as_str_name().into(),
                 state: worker.get_state().unwrap().as_str_name().into(),
-                parallelism: worker.parallel_units.len() as i32,
+                parallelism: worker.parallelism() as i32,
                 is_streaming: property.map(|p| p.is_streaming),
                 is_serving: property.map(|p| p.is_serving),
                 is_unschedulable: property.map(|p| p.is_unschedulable),
diff --git a/src/frontend/src/catalog/table_catalog.rs b/src/frontend/src/catalog/table_catalog.rs
index 29c74a64e2551..7d7d05fe92eef 100644
--- a/src/frontend/src/catalog/table_catalog.rs
+++ b/src/frontend/src/catalog/table_catalog.rs
@@ -464,6 +464,24 @@ impl TableCatalog {
         }
     }
 
+    pub fn default_column_exprs(columns: &[ColumnCatalog]) -> Vec<ExprImpl> {
+        columns
+            .iter()
+            .map(|c| {
+                if let Some(GeneratedOrDefaultColumn::DefaultColumn(DefaultColumnDesc {
+                    expr,
+                    ..
+                })) = c.column_desc.generated_or_default_column.as_ref()
+                {
+                    ExprImpl::from_expr_proto(expr.as_ref().unwrap())
+                        .expect("expr in default columns corrupted")
+                } else {
+                    ExprImpl::literal_null(c.data_type().clone())
+                }
+            })
+            .collect()
+    }
+
     pub fn default_columns(&self) -> impl Iterator<Item = (usize, ExprImpl)> + '_ {
         self.columns.iter().enumerate().filter_map(|(i, c)| {
             if let Some(GeneratedOrDefaultColumn::DefaultColumn(DefaultColumnDesc {
diff --git a/src/frontend/src/catalog/view_catalog.rs b/src/frontend/src/catalog/view_catalog.rs
index a884eed3d0e2a..331613be9415d 100644
--- a/src/frontend/src/catalog/view_catalog.rs
+++ b/src/frontend/src/catalog/view_catalog.rs
@@ -36,7 +36,7 @@ impl From<&PbView> for ViewCatalog {
             id: view.id,
             name: view.name.clone(),
             owner: view.owner,
-            properties: WithOptions::new(view.properties.clone()),
+            properties: WithOptions::new_with_options(view.properties.clone()),
             sql: view.sql.clone(),
             columns: view.columns.iter().map(|f| f.into()).collect(),
         }
diff --git a/src/frontend/src/error.rs b/src/frontend/src/error.rs
index 93b3e627ae856..3092c9bee91a9 100644
--- a/src/frontend/src/error.rs
+++ b/src/frontend/src/error.rs
@@ -15,6 +15,7 @@
 use risingwave_batch::error::BatchError;
 use risingwave_common::array::ArrayError;
 use risingwave_common::error::{BoxedError, NoFunction, NotImplemented};
+use risingwave_common::secret::SecretError;
 use risingwave_common::session_config::SessionConfigError;
 use risingwave_common::util::value_encoding::error::ValueEncodingError;
 use risingwave_connector::error::ConnectorError;
@@ -164,6 +165,12 @@ pub enum ErrorCode {
         #[backtrace]
         SessionConfigError,
     ),
+    #[error("Secret error: {0}")]
+    SecretError(
+        #[from]
+        #[backtrace]
+        SecretError,
+    ),
     #[error("{0} has been deprecated, please use {1} instead.")]
     Deprecated(String, String),
 }
diff --git a/src/frontend/src/expr/agg_call.rs b/src/frontend/src/expr/agg_call.rs
index d72fba4dbcd2c..452d37652d341 100644
--- a/src/frontend/src/expr/agg_call.rs
+++ b/src/frontend/src/expr/agg_call.rs
@@ -12,13 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::sync::Arc;
-
 use risingwave_common::types::DataType;
 use risingwave_expr::aggregate::AggKind;
 
 use super::{infer_type, Expr, ExprImpl, Literal, OrderBy};
-use crate::catalog::function_catalog::{FunctionCatalog, FunctionKind};
 use crate::error::Result;
 use crate::utils::Condition;
 
@@ -31,8 +28,6 @@ pub struct AggCall {
     pub order_by: OrderBy,
     pub filter: Condition,
     pub direct_args: Vec<Literal>,
-    /// Catalog of user defined aggregate function.
-    pub user_defined: Option<Arc<FunctionCatalog>>,
 }
 
 impl std::fmt::Debug for AggCall {
@@ -43,6 +38,9 @@ impl std::fmt::Debug for AggCall {
                 .field("return_type", &self.return_type)
                 .field("args", &self.args)
                 .field("filter", &self.filter)
+                .field("distinct", &self.distinct)
+                .field("order_by", &self.order_by)
+                .field("direct_args", &self.direct_args)
                 .finish()
         } else {
             let mut builder = f.debug_tuple(&format!("{}", self.agg_kind));
@@ -65,7 +63,11 @@ impl AggCall {
         filter: Condition,
         direct_args: Vec<Literal>,
     ) -> Result<Self> {
-        let return_type = infer_type(agg_kind.into(), &mut args)?;
+        let return_type = match &agg_kind {
+            AggKind::Builtin(kind) => infer_type((*kind).into(), &mut args)?,
+            AggKind::UserDefined(udf) => udf.return_type.as_ref().unwrap().into(),
+            AggKind::WrapScalar(expr) => expr.return_type.as_ref().unwrap().into(),
+        };
         Ok(AggCall {
             agg_kind,
             return_type,
@@ -74,7 +76,6 @@ impl AggCall {
             order_by,
             filter,
             direct_args,
-            user_defined: None,
         })
     }
 
@@ -92,36 +93,11 @@ impl AggCall {
             order_by: OrderBy::any(),
             filter: Condition::true_cond(),
             direct_args: vec![],
-            user_defined: None,
-        })
-    }
-
-    /// Create a user-defined `AggCall`.
-    pub fn new_user_defined(
-        args: Vec<ExprImpl>,
-        distinct: bool,
-        order_by: OrderBy,
-        filter: Condition,
-        direct_args: Vec<Literal>,
-        user_defined: Arc<FunctionCatalog>,
-    ) -> Result<Self> {
-        let FunctionKind::Aggregate = &user_defined.kind else {
-            panic!("not an aggregate function");
-        };
-        Ok(AggCall {
-            agg_kind: AggKind::UserDefined,
-            return_type: user_defined.return_type.clone(),
-            args,
-            distinct,
-            order_by,
-            filter,
-            direct_args,
-            user_defined: Some(user_defined),
         })
     }
 
     pub fn agg_kind(&self) -> AggKind {
-        self.agg_kind
+        self.agg_kind.clone()
     }
 
     /// Get a reference to the agg call's arguments.
diff --git a/src/frontend/src/expr/expr_rewriter.rs b/src/frontend/src/expr/expr_rewriter.rs
index 6300f9d5e8858..ccbf1b329bba5 100644
--- a/src/frontend/src/expr/expr_rewriter.rs
+++ b/src/frontend/src/expr/expr_rewriter.rs
@@ -94,7 +94,6 @@ pub trait ExprRewriter {
             order_by,
             filter,
             direct_args,
-            user_defined,
         } = agg_call;
         let args = args
             .into_iter()
@@ -110,7 +109,6 @@ pub trait ExprRewriter {
             order_by,
             filter,
             direct_args,
-            user_defined,
         }
         .into()
     }
diff --git a/src/frontend/src/expr/function_impl/context.rs b/src/frontend/src/expr/function_impl/context.rs
index 74cc5001043ad..4aca6219051a1 100644
--- a/src/frontend/src/expr/function_impl/context.rs
+++ b/src/frontend/src/expr/function_impl/context.rs
@@ -26,4 +26,5 @@ define_context! {
     pub(super) AUTH_CONTEXT: Arc<AuthContext>,
     pub(super) DB_NAME: String,
     pub(super) SEARCH_PATH: SearchPath,
+    pub(super) META_CLIENT: Arc<dyn crate::meta_client::FrontendMetaClient>,
 }
diff --git a/src/frontend/src/expr/function_impl/mod.rs b/src/frontend/src/expr/function_impl/mod.rs
index a0cff36840b42..4a1e0600dbc0d 100644
--- a/src/frontend/src/expr/function_impl/mod.rs
+++ b/src/frontend/src/expr/function_impl/mod.rs
@@ -22,3 +22,4 @@ mod pg_get_viewdef;
 mod pg_index_column_has_property;
 mod pg_indexes_size;
 mod pg_relation_size;
+mod rw_recovery_status;
diff --git a/src/frontend/src/expr/function_impl/rw_recovery_status.rs b/src/frontend/src/expr/function_impl/rw_recovery_status.rs
new file mode 100644
index 0000000000000..746dce01e694a
--- /dev/null
+++ b/src/frontend/src/expr/function_impl/rw_recovery_status.rs
@@ -0,0 +1,52 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::fmt::Write;
+use std::sync::Arc;
+
+use risingwave_expr::{capture_context, function, ExprError, Result};
+use risingwave_pb::meta::RecoveryStatus;
+
+use super::context::META_CLIENT;
+use crate::meta_client::FrontendMetaClient;
+
+#[function("rw_recovery_status() -> varchar", volatile)]
+async fn rw_recovery_status(writer: &mut impl Write) -> Result<()> {
+    writer
+        .write_str(
+            rw_recovery_status_impl_captured()
+                .await?
+                .as_str_name()
+                .strip_prefix("STATUS_")
+                .unwrap(),
+        )
+        .unwrap();
+    Ok(())
+}
+
+#[function("pg_is_in_recovery() -> boolean", volatile)]
+async fn pg_is_in_recovery() -> Result<bool> {
+    let status = rw_recovery_status_impl_captured().await?;
+    Ok(status != RecoveryStatus::StatusRunning)
+}
+
+#[capture_context(META_CLIENT)]
+async fn rw_recovery_status_impl(
+    meta_client: &Arc<dyn FrontendMetaClient>,
+) -> Result<RecoveryStatus> {
+    meta_client
+        .get_cluster_recovery_status()
+        .await
+        .map_err(|e| ExprError::Internal(e.into()))
+}
diff --git a/src/frontend/src/expr/mod.rs b/src/frontend/src/expr/mod.rs
index 89142d0e9b237..d42317b00f10b 100644
--- a/src/frontend/src/expr/mod.rs
+++ b/src/frontend/src/expr/mod.rs
@@ -18,7 +18,7 @@ use futures::FutureExt;
 use paste::paste;
 use risingwave_common::array::ListValue;
 use risingwave_common::types::{DataType, Datum, JsonbVal, Scalar, ScalarImpl};
-use risingwave_expr::aggregate::AggKind;
+use risingwave_expr::aggregate::PbAggKind;
 use risingwave_expr::expr::build_from_prost;
 use risingwave_pb::expr::expr_node::RexNode;
 use risingwave_pb::expr::{ExprNode, ProjectSetSelectItem};
@@ -94,6 +94,14 @@ macro_rules! impl_expr_impl {
             $($t(Box<$t>),)*
         }
 
+        impl ExprImpl {
+            pub fn variant_name(&self) -> &'static str {
+                match self {
+                    $(ExprImpl::$t(_) => stringify!($t),)*
+                }
+            }
+        }
+
         $(
         impl From<$t> for ExprImpl {
             fn from(o: $t) -> ExprImpl {
@@ -195,7 +203,7 @@ impl ExprImpl {
     #[inline(always)]
     pub fn count_star() -> Self {
         AggCall::new(
-            AggKind::Count,
+            PbAggKind::Count.into(),
             vec![],
             false,
             OrderBy::any(),
@@ -760,45 +768,36 @@ impl ExprImpl {
         }
     }
 
-    /// Accepts expressions of the form `input_expr cmp now() [+- const_expr]` or
-    /// `now() [+- const_expr] cmp input_expr`, where `input_expr` contains an
-    /// `InputRef` and contains no `now()`.
+    /// Accepts expressions of the form `input_expr cmp now_expr` or `now_expr cmp input_expr`,
+    /// where `input_expr` contains an `InputRef` and contains no `now()`, and `now_expr`
+    /// contains a `now()` but no `InputRef`.
     ///
     /// Canonicalizes to the first ordering and returns `(input_expr, cmp, now_expr)`
     pub fn as_now_comparison_cond(&self) -> Option<(ExprImpl, ExprType, ExprImpl)> {
         if let ExprImpl::FunctionCall(function_call) = self {
             match function_call.func_type() {
-                ty @ (ExprType::LessThan
+                ty @ (ExprType::Equal
+                | ExprType::LessThan
                 | ExprType::LessThanOrEqual
                 | ExprType::GreaterThan
                 | ExprType::GreaterThanOrEqual) => {
                     let (_, op1, op2) = function_call.clone().decompose_as_binary();
-                    if op1.count_nows() == 0
+                    if !op1.has_now()
                         && op1.has_input_ref()
-                        && op2.count_nows() > 0
-                        && op2.is_now_offset()
+                        && op2.has_now()
+                        && !op2.has_input_ref()
                     {
                         Some((op1, ty, op2))
-                    } else if op2.count_nows() == 0
+                    } else if op1.has_now()
+                        && !op1.has_input_ref()
+                        && !op2.has_now()
                         && op2.has_input_ref()
-                        && op1.count_nows() > 0
-                        && op1.is_now_offset()
                     {
                         Some((op2, Self::reverse_comparison(ty), op1))
                     } else {
                         None
                     }
                 }
-                ty @ ExprType::Equal => {
-                    let (_, op1, op2) = function_call.clone().decompose_as_binary();
-                    if op1.count_nows() == 0 && op1.has_input_ref() && op2.count_nows() > 0 {
-                        Some((op1, ty, op2))
-                    } else if op2.count_nows() == 0 && op2.has_input_ref() && op1.count_nows() > 0 {
-                        Some((op2, Self::reverse_comparison(ty), op1))
-                    } else {
-                        None
-                    }
-                }
                 _ => None,
             }
         } else {
@@ -854,23 +853,6 @@ impl ExprImpl {
         }
     }
 
-    /// Checks if expr is of the form `now() [+- const_expr]`
-    fn is_now_offset(&self) -> bool {
-        if let ExprImpl::Now(_) = self {
-            true
-        } else if let ExprImpl::FunctionCall(f) = self {
-            match f.func_type() {
-                ExprType::Add | ExprType::Subtract => {
-                    let (_, lhs, rhs) = f.clone().decompose_as_binary();
-                    lhs.is_now_offset() && rhs.is_const()
-                }
-                _ => false,
-            }
-        } else {
-            false
-        }
-    }
-
     /// Returns the `InputRef` and offset of a predicate if it matches
     /// the form `InputRef [+- const_expr]`, else returns None.
     fn as_input_offset(&self) -> Option<(usize, Option<(ExprType, ExprImpl)>)> {
diff --git a/src/frontend/src/expr/pure.rs b/src/frontend/src/expr/pure.rs
index fe87eb6c2280c..dd2f353a34b0d 100644
--- a/src/frontend/src/expr/pure.rs
+++ b/src/frontend/src/expr/pure.rs
@@ -135,6 +135,7 @@ impl ExprVisitor for ImpureAnalyzer {
             | Type::Cot
             | Type::Asin
             | Type::Acos
+            | Type::Acosd
             | Type::Atan
             | Type::Atan2
             | Type::Sqrt
@@ -275,7 +276,9 @@ impl ExprVisitor for ImpureAnalyzer {
             | Type::HasTablePrivilege
             | Type::HasAnyColumnPrivilege
             | Type::HasSchemaPrivilege
-            | Type::MakeTimestamptz => self.impure = true,
+            | Type::MakeTimestamptz
+            | Type::PgIsInRecovery
+            | Type::RwRecoveryStatus => self.impure = true,
         }
     }
 }
diff --git a/src/frontend/src/expr/table_function.rs b/src/frontend/src/expr/table_function.rs
index 0db14d4736c2d..5806eea792904 100644
--- a/src/frontend/src/expr/table_function.rs
+++ b/src/frontend/src/expr/table_function.rs
@@ -12,15 +12,19 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::sync::Arc;
+use std::sync::{Arc, LazyLock};
 
 use itertools::Itertools;
-use risingwave_common::types::DataType;
+use risingwave_common::array::arrow::IcebergArrowConvert;
+use risingwave_common::types::{DataType, ScalarImpl, StructType};
+use risingwave_connector::source::iceberg::{create_parquet_stream_builder, list_s3_directory};
 pub use risingwave_pb::expr::table_function::PbType as TableFunctionType;
 use risingwave_pb::expr::PbTableFunction;
+use tokio::runtime::Runtime;
 
-use super::{infer_type, Expr, ExprImpl, ExprRewriter, RwResult};
+use super::{infer_type, Expr, ExprImpl, ExprRewriter, Literal, RwResult};
 use crate::catalog::function_catalog::{FunctionCatalog, FunctionKind};
+use crate::error::ErrorCode::BindError;
 
 /// A table function takes a row as input and returns a table. It is also known as Set-Returning
 /// Function.
@@ -62,6 +66,166 @@ impl TableFunction {
         }
     }
 
+    /// A special table function which would be transformed into `LogicalFileScan` by `TableFunctionToFileScanRule` in the optimizer.
+    /// select * from `file_scan`('parquet', 's3', region, ak, sk, location)
+    pub fn new_file_scan(mut args: Vec<ExprImpl>) -> RwResult<Self> {
+        let return_type = {
+            // arguments:
+            // file format e.g. parquet
+            // storage type e.g. s3
+            // s3 region
+            // s3 access key
+            // s3 secret key
+            // file location
+            if args.len() != 6 {
+                return Err(BindError("file_scan function only accepts 6 arguments: file_scan('parquet', 's3', s3 region, s3 access key, s3 secret key, file location)".to_string()).into());
+            }
+            let mut eval_args: Vec<String> = vec![];
+            for arg in &args {
+                if arg.return_type() != DataType::Varchar {
+                    return Err(BindError(
+                        "file_scan function only accepts string arguments".to_string(),
+                    )
+                    .into());
+                }
+                match arg.try_fold_const() {
+                    Some(Ok(value)) => {
+                        if value.is_none() {
+                            return Err(BindError(
+                                "file_scan function does not accept null arguments".to_string(),
+                            )
+                            .into());
+                        }
+                        match value {
+                            Some(ScalarImpl::Utf8(s)) => {
+                                eval_args.push(s.to_string());
+                            }
+                            _ => {
+                                return Err(BindError(
+                                    "file_scan function only accepts string arguments".to_string(),
+                                )
+                                .into())
+                            }
+                        }
+                    }
+                    Some(Err(err)) => {
+                        return Err(err);
+                    }
+                    None => {
+                        return Err(BindError(
+                            "file_scan function only accepts constant arguments".to_string(),
+                        )
+                        .into());
+                    }
+                }
+            }
+            if !"parquet".eq_ignore_ascii_case(&eval_args[0]) {
+                return Err(BindError(
+                    "file_scan function only accepts 'parquet' as file format".to_string(),
+                )
+                .into());
+            }
+
+            if !"s3".eq_ignore_ascii_case(&eval_args[1]) {
+                return Err(BindError(
+                    "file_scan function only accepts 's3' as storage type".to_string(),
+                )
+                .into());
+            }
+
+            #[cfg(madsim)]
+            return Err(crate::error::ErrorCode::BindError(
+                "file_scan can't be used in the madsim mode".to_string(),
+            )
+            .into());
+
+            #[cfg(not(madsim))]
+            {
+                static RUNTIME: LazyLock<Runtime> = LazyLock::new(|| {
+                    tokio::runtime::Builder::new_multi_thread()
+                        .thread_name("rw-file-scan")
+                        .enable_all()
+                        .build()
+                        .expect("failed to build file-scan runtime")
+                });
+
+                let files = if eval_args[5].ends_with('/') {
+                    let files = tokio::task::block_in_place(|| {
+                        RUNTIME.block_on(async {
+                            let files = list_s3_directory(
+                                eval_args[2].clone(),
+                                eval_args[3].clone(),
+                                eval_args[4].clone(),
+                                eval_args[5].clone(),
+                            )
+                            .await?;
+
+                            Ok::<Vec<String>, anyhow::Error>(files)
+                        })
+                    })?;
+
+                    if files.is_empty() {
+                        return Err(BindError(
+                            "file_scan function only accepts non-empty directory".to_string(),
+                        )
+                        .into());
+                    }
+
+                    Some(files)
+                } else {
+                    None
+                };
+
+                let schema = tokio::task::block_in_place(|| {
+                    RUNTIME.block_on(async {
+                        let parquet_stream_builder = create_parquet_stream_builder(
+                            eval_args[2].clone(),
+                            eval_args[3].clone(),
+                            eval_args[4].clone(),
+                            match files.as_ref() {
+                                Some(files) => files[0].clone(),
+                                None => eval_args[5].clone(),
+                            },
+                        )
+                        .await?;
+
+                        let mut rw_types = vec![];
+                        for field in parquet_stream_builder.schema().fields() {
+                            rw_types.push((
+                                field.name().to_string(),
+                                IcebergArrowConvert.type_from_field(field)?,
+                            ));
+                        }
+
+                        Ok::<risingwave_common::types::DataType, anyhow::Error>(DataType::Struct(
+                            StructType::new(rw_types),
+                        ))
+                    })
+                })?;
+
+                if let Some(files) = files {
+                    // if the file location is a directory, we need to remove the last argument and add all files in the directory as arguments
+                    args.remove(5);
+                    for file in files {
+                        args.push(ExprImpl::Literal(Box::new(Literal::new(
+                            Some(ScalarImpl::Utf8(file.into())),
+                            DataType::Varchar,
+                        ))));
+                    }
+                }
+
+                schema
+            }
+        };
+
+        Ok(TableFunction {
+            args,
+            return_type,
+            function_type: TableFunctionType::FileScan,
+            user_defined: None,
+        })
+    }
+
     pub fn to_protobuf(&self) -> PbTableFunction {
         PbTableFunction {
             function_type: self.function_type as i32,
diff --git a/src/frontend/src/expr/type_inference/func.rs b/src/frontend/src/expr/type_inference/func.rs
index 0fecee8ab45c0..5b6a12ba58cce 100644
--- a/src/frontend/src/expr/type_inference/func.rs
+++ b/src/frontend/src/expr/type_inference/func.rs
@@ -18,8 +18,8 @@ use risingwave_common::bail_no_function;
 use risingwave_common::hash::VirtualNode;
 use risingwave_common::types::{DataType, StructType};
 use risingwave_common::util::iter_util::ZipEqFast;
-use risingwave_expr::aggregate::AggKind;
 pub use risingwave_expr::sig::*;
+use risingwave_pb::expr::agg_call::PbType as PbAggKind;
 use risingwave_pb::expr::table_function::PbType as PbTableFuncType;
 
 use super::{align_types, cast_ok_base, CastContext};
@@ -50,7 +50,7 @@ pub fn infer_type_with_sigmap(
             }
         }
         FuncName::Aggregate(agg_kind) => {
-            if *agg_kind == AggKind::Grouping {
+            if *agg_kind == PbAggKind::Grouping {
                 return Ok(DataType::Int32);
             }
         }
diff --git a/src/frontend/src/expr/utils.rs b/src/frontend/src/expr/utils.rs
index cc49f3c215378..b5f68fed4dfbe 100644
--- a/src/frontend/src/expr/utils.rs
+++ b/src/frontend/src/expr/utils.rs
@@ -508,191 +508,6 @@ pub fn rewrite_now_to_proctime(expr: ExprImpl) -> ExprImpl {
     r.rewrite_expr(expr)
 }
 
-/// analyze if the expression can derive a watermark from some input watermark. If it can
-/// derive, return the input watermark column index
-pub fn try_derive_watermark(expr: &ExprImpl) -> WatermarkDerivation {
-    let a = WatermarkAnalyzer {};
-    a.visit_expr(expr)
-}
-
-#[derive(Clone, Copy, Debug, PartialEq)]
-pub enum WatermarkDerivation {
-    /// The expression will return a constant and not depends on its input.
-    Constant,
-    /// Can derive a watermark if an input column has watermark, the usize field is the input
-    /// column index.
-    Watermark(usize),
-    /// For nondecreasing functions, we can always produce watermarks from where they are called.
-    Nondecreasing,
-    /// Can not derive a watermark in any cases.
-    None,
-}
-
-#[derive(Clone, Default)]
-struct WatermarkAnalyzer {}
-
-impl WatermarkAnalyzer {
-    fn visit_expr(&self, expr: &ExprImpl) -> WatermarkDerivation {
-        match expr {
-            ExprImpl::InputRef(inner) => WatermarkDerivation::Watermark(inner.index()),
-            ExprImpl::Literal(_) => WatermarkDerivation::Constant,
-            ExprImpl::FunctionCall(inner) => self.visit_function_call(inner),
-            ExprImpl::FunctionCallWithLambda(inner) => self.visit_function_call(inner.base()),
-            ExprImpl::TableFunction(_) => WatermarkDerivation::None,
-            ExprImpl::Subquery(_)
-            | ExprImpl::AggCall(_)
-            | ExprImpl::CorrelatedInputRef(_)
-            | ExprImpl::WindowFunction(_)
-            | ExprImpl::Parameter(_)
-            | ExprImpl::Now(_) => unreachable!(),
-            ExprImpl::UserDefinedFunction(_) => WatermarkDerivation::None,
-        }
-    }
-
-    fn visit_unary_op(&self, inputs: &[ExprImpl]) -> WatermarkDerivation {
-        assert_eq!(inputs.len(), 1);
-        self.visit_expr(&inputs[0])
-    }
-
-    fn visit_binary_op(&self, inputs: &[ExprImpl]) -> (WatermarkDerivation, WatermarkDerivation) {
-        assert_eq!(inputs.len(), 2);
-        (self.visit_expr(&inputs[0]), self.visit_expr(&inputs[1]))
-    }
-
-    fn visit_ternary_op(
-        &self,
-        inputs: &[ExprImpl],
-    ) -> (
-        WatermarkDerivation,
-        WatermarkDerivation,
-        WatermarkDerivation,
-    ) {
-        assert_eq!(inputs.len(), 3);
-        (
-            self.visit_expr(&inputs[0]),
-            self.visit_expr(&inputs[1]),
-            self.visit_expr(&inputs[2]),
-        )
-    }
-
-    fn visit_function_call(&self, func_call: &FunctionCall) -> WatermarkDerivation {
-        use WatermarkDerivation::{Constant, Nondecreasing, Watermark};
-        match func_call.func_type() {
-            ExprType::Unspecified => unreachable!(),
-            ExprType::Add => match self.visit_binary_op(func_call.inputs()) {
-                (Constant, Constant) => Constant,
-                (Constant, Watermark(idx)) | (Watermark(idx), Constant) => Watermark(idx),
-                (Constant, Nondecreasing) | (Nondecreasing, Constant) => Nondecreasing,
-                _ => WatermarkDerivation::None,
-            },
-            ExprType::Subtract | ExprType::TumbleStart => {
-                if func_call.inputs().len() == 3 {
-                    // With `offset` specified
-                    // e.g., select * from tumble(t1, start, interval, offset);
-                    assert_eq!(ExprType::TumbleStart, func_call.func_type());
-                    match self.visit_ternary_op(func_call.inputs()) {
-                        (Constant, Constant, Constant) => Constant,
-                        (Watermark(idx), Constant, Constant) => Watermark(idx),
-                        (Nondecreasing, Constant, Constant) => Nondecreasing,
-                        _ => WatermarkDerivation::None,
-                    }
-                } else {
-                    match self.visit_binary_op(func_call.inputs()) {
-                        (Constant, Constant) => Constant,
-                        (Watermark(idx), Constant) => Watermark(idx),
-                        (Nondecreasing, Constant) => Nondecreasing,
-                        _ => WatermarkDerivation::None,
-                    }
-                }
-            }
-            ExprType::Multiply | ExprType::Divide | ExprType::Modulus => {
-                match self.visit_binary_op(func_call.inputs()) {
-                    (Constant, Constant) => Constant,
-                    // not meaningful to derive watermark for other situations
-                    _ => WatermarkDerivation::None,
-                }
-            }
-            ExprType::AtTimeZone => match self.visit_binary_op(func_call.inputs()) {
-                (Constant, Constant) => Constant,
-                (derivation @ (Watermark(_) | Nondecreasing), Constant) => {
-                    if !(func_call.return_type() == DataType::Timestamptz
-                        && func_call.inputs()[0].return_type() == DataType::Timestamp)
-                        && func_call.inputs()[1]
-                            .as_literal()
-                            .and_then(|literal| literal.get_data().as_ref())
-                            .map_or(true, |time_zone| {
-                                !time_zone.as_utf8().eq_ignore_ascii_case("UTC")
-                            })
-                    {
-                        WatermarkDerivation::None
-                    } else {
-                        derivation
-                    }
-                }
-                _ => WatermarkDerivation::None,
-            },
-            ExprType::AddWithTimeZone | ExprType::SubtractWithTimeZone => {
-                // Requires time zone and interval to be literal, at least for now.
-                let time_zone = match &func_call.inputs()[2] {
-                    ExprImpl::Literal(lit) => lit.get_data().as_ref().map(|s| s.as_utf8()),
-                    _ => return WatermarkDerivation::None,
-                };
-                let interval = match &func_call.inputs()[1] {
-                    ExprImpl::Literal(lit) => lit.get_data().as_ref().map(|s| s.as_interval()),
-                    _ => return WatermarkDerivation::None,
-                };
-                // null zone or null interval is treated same as const `interval '1' second`, to be
-                // consistent with other match arms.
-                let zone_without_dst = time_zone.map_or(true, |s| s.eq_ignore_ascii_case("UTC"));
-                let quantitative_only = interval.map_or(true, |v| {
-                    v.months() == 0 && (v.days() == 0 || zone_without_dst)
-                });
-                match (self.visit_expr(&func_call.inputs()[0]), quantitative_only) {
-                    (Constant, _) => Constant,
-                    (Watermark(idx), true) => Watermark(idx),
-                    (Nondecreasing, true) => Nondecreasing,
-                    (Watermark(_) | Nondecreasing, false) => WatermarkDerivation::None,
-                    (WatermarkDerivation::None, _) => WatermarkDerivation::None,
-                }
-            }
-            ExprType::DateTrunc => match func_call.inputs().len() {
-                2 => match self.visit_binary_op(func_call.inputs()) {
-                    (Constant, any_derivation) => any_derivation,
-                    _ => WatermarkDerivation::None,
-                },
-                3 => match self.visit_ternary_op(func_call.inputs()) {
-                    (Constant, Constant, Constant) => Constant,
-                    (Constant, derivation @ (Watermark(_) | Nondecreasing), Constant) => {
-                        let zone_without_dst = func_call.inputs()[2]
-                            .as_literal()
-                            .and_then(|literal| literal.get_data().as_ref())
-                            .map_or(false, |s| s.as_utf8().eq_ignore_ascii_case("UTC"));
-                        if zone_without_dst {
-                            derivation
-                        } else {
-                            WatermarkDerivation::None
-                        }
-                    }
-                    _ => WatermarkDerivation::None,
-                },
-                _ => unreachable!(),
-            },
-            ExprType::SecToTimestamptz => self.visit_unary_op(func_call.inputs()),
-            ExprType::CharToTimestamptz => WatermarkDerivation::None,
-            ExprType::Cast => {
-                // TODO: need more derivation
-                WatermarkDerivation::None
-            }
-            ExprType::Case => {
-                // TODO: do we need derive watermark when every case can derive a common watermark?
-                WatermarkDerivation::None
-            }
-            ExprType::Proctime => Nondecreasing,
-            _ => WatermarkDerivation::None,
-        }
-    }
-}
-
 #[cfg(test)]
 mod tests {
     use risingwave_common::types::{DataType, ScalarImpl};
diff --git a/src/frontend/src/expr/window_function.rs b/src/frontend/src/expr/window_function.rs
index fd9c38961132b..70f6a79866fb3 100644
--- a/src/frontend/src/expr/window_function.rs
+++ b/src/frontend/src/expr/window_function.rs
@@ -15,6 +15,7 @@
 use itertools::Itertools;
 use risingwave_common::bail_not_implemented;
 use risingwave_common::types::DataType;
+use risingwave_expr::aggregate::AggKind;
 use risingwave_expr::sig::FUNCTION_REGISTRY;
 use risingwave_expr::window_function::{Frame, WindowFuncKind};
 
@@ -47,7 +48,7 @@ impl WindowFunction {
         args: Vec<ExprImpl>,
         frame: Option<Frame>,
     ) -> RwResult<Self> {
-        let return_type = Self::infer_return_type(kind, &args)?;
+        let return_type = Self::infer_return_type(&kind, &args)?;
         Ok(Self {
             kind,
             args,
@@ -58,7 +59,7 @@ impl WindowFunction {
         })
     }
 
-    fn infer_return_type(kind: WindowFuncKind, args: &[ExprImpl]) -> RwResult<DataType> {
+    fn infer_return_type(kind: &WindowFuncKind, args: &[ExprImpl]) -> RwResult<DataType> {
         use WindowFuncKind::*;
         match (kind, args) {
             (RowNumber, []) => Ok(DataType::Int64),
@@ -86,9 +87,9 @@ impl WindowFunction {
                 );
             }
 
-            (Aggregate(agg_kind), args) => {
+            (Aggregate(AggKind::Builtin(agg_kind)), args) => {
                 let arg_types = args.iter().map(ExprImpl::return_type).collect::<Vec<_>>();
-                let return_type = FUNCTION_REGISTRY.get_return_type(agg_kind, &arg_types)?;
+                let return_type = FUNCTION_REGISTRY.get_return_type(*agg_kind, &arg_types)?;
                 Ok(return_type)
             }
 
diff --git a/src/frontend/src/handler/alter_source_with_sr.rs b/src/frontend/src/handler/alter_source_with_sr.rs
index 070b26b6a25e2..01e09b0c4f5f5 100644
--- a/src/frontend/src/handler/alter_source_with_sr.rs
+++ b/src/frontend/src/handler/alter_source_with_sr.rs
@@ -15,6 +15,7 @@
 use std::sync::Arc;
 
 use anyhow::Context;
+use either::Either;
 use itertools::Itertools;
 use pgwire::pg_response::StatementType;
 use risingwave_common::bail_not_implemented;
@@ -37,6 +38,7 @@ use crate::catalog::source_catalog::SourceCatalog;
 use crate::catalog::{DatabaseId, SchemaId};
 use crate::error::{ErrorCode, Result};
 use crate::session::SessionImpl;
+use crate::utils::resolve_secret_ref_in_with_options;
 use crate::{Binder, WithOptions};
 
 fn format_type_to_format(from: FormatType) -> Option<Format> {
@@ -63,6 +65,7 @@ fn encode_type_to_encode(from: EncodeType) -> Option<Encode> {
         EncodeType::Json => Encode::Json,
         EncodeType::Bytes => Encode::Bytes,
         EncodeType::Template => Encode::Template,
+        EncodeType::Parquet => Encode::Parquet,
         EncodeType::None => Encode::None,
         EncodeType::Text => Encode::Text,
     })
@@ -161,7 +164,8 @@ pub async fn refresh_sr_and_get_columns_diff(
     }
 
     let (Some(columns_from_resolve_source), source_info) =
-        bind_columns_from_source(session, connector_schema, &with_properties).await?
+        bind_columns_from_source(session, connector_schema, Either::Right(&with_properties))
+            .await?
     else {
         // Source without schema registry is rejected.
         unreachable!("source without schema registry is rejected")
@@ -253,12 +257,21 @@ pub async fn handle_alter_source_with_sr(
     source.definition =
         alter_definition_format_encode(&source.definition, connector_schema.row_options.clone())?;
 
-    let format_encode_options = WithOptions::try_from(connector_schema.row_options())?.into_inner();
+    let (format_encode_options, format_encode_secret_ref) = resolve_secret_ref_in_with_options(
+        WithOptions::try_from(connector_schema.row_options())?,
+        session.as_ref(),
+    )?
+    .into_parts();
     source
         .info
         .format_encode_options
         .extend(format_encode_options);
 
+    source
+        .info
+        .format_encode_secret_refs
+        .extend(format_encode_secret_ref);
+
     let mut pb_source = source.to_prost(schema_id, database_id);
 
     // update version
diff --git a/src/frontend/src/handler/alter_streaming_rate_limit.rs b/src/frontend/src/handler/alter_streaming_rate_limit.rs
index e5273b85d57b8..2dff4338a1de3 100644
--- a/src/frontend/src/handler/alter_streaming_rate_limit.rs
+++ b/src/frontend/src/handler/alter_streaming_rate_limit.rs
@@ -42,7 +42,7 @@ pub async fn handle_alter_streaming_rate_limit(
         PbThrottleTarget::Mv => {
             let reader = session.env().catalog_reader().read_guard();
             let (table, schema_name) =
-                reader.get_created_table_by_name(db_name, schema_path, &real_table_name)?;
+                reader.get_any_table_by_name(db_name, schema_path, &real_table_name)?;
             if table.table_type != TableType::MaterializedView {
                 return Err(ErrorCode::InvalidInputSyntax(format!(
                     "\"{table_name}\" is not a materialized view",
@@ -68,7 +68,7 @@ pub async fn handle_alter_streaming_rate_limit(
             let source_id = if let Some(id) = table.associated_source_id {
                 id.table_id()
             } else {
-                bail!("ALTER STREAMING_RATE_LIMIT is not for table without source")
+                bail!("ALTER SOURCE_RATE_LIMIT is not for table without source")
             };
             (StatementType::ALTER_SOURCE, source_id)
         }
diff --git a/src/frontend/src/handler/alter_table_column.rs b/src/frontend/src/handler/alter_table_column.rs
index 3d9c5d71b4576..ba3ae95399783 100644
--- a/src/frontend/src/handler/alter_table_column.rs
+++ b/src/frontend/src/handler/alter_table_column.rs
@@ -12,13 +12,18 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::collections::HashSet;
 use std::sync::Arc;
 
 use anyhow::Context;
 use itertools::Itertools;
 use pgwire::pg_response::{PgResponse, StatementType};
-use risingwave_common::bail_not_implemented;
+use risingwave_common::catalog::ColumnCatalog;
 use risingwave_common::util::column_index_mapping::ColIndexMapping;
+use risingwave_common::{bail, bail_not_implemented};
+use risingwave_connector::sink::catalog::SinkCatalog;
+use risingwave_pb::stream_plan::stream_node::PbNodeBody;
+use risingwave_pb::stream_plan::{ProjectNode, StreamFragmentGraph};
 use risingwave_sqlparser::ast::{
     AlterTableOperation, ColumnOption, ConnectorSchema, Encode, ObjectName, Statement,
 };
@@ -31,6 +36,8 @@ use super::{HandlerArgs, RwPgResponse};
 use crate::catalog::root_catalog::SchemaPath;
 use crate::catalog::table_catalog::TableType;
 use crate::error::{ErrorCode, Result, RwError};
+use crate::expr::{Expr, ExprImpl, InputRef, Literal};
+use crate::handler::create_sink::{fetch_incoming_sinks, insert_merger_to_union_with_project};
 use crate::session::SessionImpl;
 use crate::{Binder, TableCatalog, WithOptions};
 
@@ -52,26 +59,28 @@ pub async fn replace_table_with_definition(
         on_conflict,
         with_version_column,
         wildcard_idx,
+        cdc_table_info,
         ..
     } = definition
     else {
         panic!("unexpected statement type: {:?}", definition);
     };
 
-    let (graph, table, source) = generate_stream_graph_for_table(
+    let (mut graph, mut table, source, job_type) = generate_stream_graph_for_table(
         session,
         table_name,
         original_catalog,
         source_schema,
-        handler_args,
+        handler_args.clone(),
         col_id_gen,
-        columns,
+        columns.clone(),
         wildcard_idx,
         constraints,
         source_watermarks,
         append_only,
         on_conflict,
         with_version_column,
+        cdc_table_info,
     )
     .await?;
 
@@ -89,14 +98,85 @@ pub async fn replace_table_with_definition(
         table.columns.len(),
     );
 
+    let incoming_sink_ids: HashSet<_> = original_catalog.incoming_sinks.iter().copied().collect();
+
+    let target_columns = table
+        .columns
+        .iter()
+        .map(|col| ColumnCatalog::from(col.clone()))
+        .collect_vec();
+
+    for sink in fetch_incoming_sinks(session, &incoming_sink_ids)? {
+        hijack_merger_for_target_table(
+            &mut graph,
+            &target_columns,
+            &sink,
+            Some(&sink.unique_identity()),
+        )?;
+    }
+
+    table.incoming_sinks = incoming_sink_ids.iter().copied().collect();
+
     let catalog_writer = session.catalog_writer()?;
 
     catalog_writer
-        .replace_table(source, table, graph, col_index_mapping)
+        .replace_table(source, table, graph, col_index_mapping, job_type)
         .await?;
     Ok(())
 }
 
+pub(crate) fn hijack_merger_for_target_table(
+    graph: &mut StreamFragmentGraph,
+    target_columns: &[ColumnCatalog],
+    sink: &SinkCatalog,
+    uniq_identify: Option<&str>,
+) -> Result<()> {
+    let mut sink_columns = sink.original_target_columns.clone();
+    if sink_columns.is_empty() {
+        // This is due to the fact that the value did not exist in earlier versions,
+        // which means no schema changes such as `ADD/DROP COLUMN` have been made to the table.
+        // Therefore the columns of the table at this point are `original_target_columns`.
+        // This value of sink will be filled on the meta.
+        sink_columns = target_columns.to_vec();
+    }
+
+    let mut i = 0;
+    let mut j = 0;
+    let mut exprs = Vec::new();
+
+    while j < target_columns.len() {
+        if i < sink_columns.len() && sink_columns[i].data_type() == target_columns[j].data_type() {
+            exprs.push(ExprImpl::InputRef(Box::new(InputRef {
+                data_type: sink_columns[i].data_type().clone(),
+                index: i,
+            })));
+
+            i += 1;
+            j += 1;
+        } else {
+            exprs.push(ExprImpl::Literal(Box::new(Literal::new(
+                None,
+                target_columns[j].data_type().clone(),
+            ))));
+
+            j += 1;
+        }
+    }
+
+    let pb_project = PbNodeBody::Project(ProjectNode {
+        select_list: exprs.iter().map(|expr| expr.to_expr_proto()).collect(),
+        ..Default::default()
+    });
+
+    for fragment in graph.fragments.values_mut() {
+        if let Some(node) = &mut fragment.node {
+            insert_merger_to_union_with_project(node, &pb_project, uniq_identify);
+        }
+    }
+
+    Ok(())
+}
+
 /// Handle `ALTER TABLE [ADD|DROP] COLUMN` statements. The `operation` must be either `AddColumn` or
 /// `DropColumn`.
 pub async fn handle_alter_table_column(
@@ -107,14 +187,10 @@ pub async fn handle_alter_table_column(
     let session = handler_args.session;
     let original_catalog = fetch_table_catalog_for_alter(session.as_ref(), &table_name)?;
 
-    if !original_catalog.incoming_sinks.is_empty() {
-        bail_not_implemented!("alter table with incoming sinks");
-    }
-
-    // TODO(yuhao): alter table with generated columns.
-    if original_catalog.has_generated_column() {
+    if !original_catalog.incoming_sinks.is_empty() && original_catalog.has_generated_column() {
         return Err(RwError::from(ErrorCode::BindError(
-            "Alter a table with generated column has not been implemented.".to_string(),
+            "Alter a table with incoming sink and generated column has not been implemented."
+                .to_string(),
         )));
     }
 
@@ -135,20 +211,40 @@ pub async fn handle_alter_table_column(
         .clone()
         .map(|source_schema| source_schema.into_v2_with_warning());
 
-    if let Some(source_schema) = &source_schema {
-        if schema_has_schema_registry(source_schema) {
-            return Err(ErrorCode::NotSupported(
+    let fail_if_has_schema_registry = || {
+        if let Some(source_schema) = &source_schema
+            && schema_has_schema_registry(source_schema)
+        {
+            Err(ErrorCode::NotSupported(
                 "alter table with schema registry".to_string(),
                 "try `ALTER TABLE .. FORMAT .. ENCODE .. (...)` instead".to_string(),
-            )
-            .into());
+            ))
+        } else {
+            Ok(())
         }
+    };
+
+    if columns.is_empty() {
+        Err(ErrorCode::NotSupported(
+            "alter a table with empty column definitions".to_string(),
+            "Please recreate the table with column definitions.".to_string(),
+        ))?
+    }
+
+    if !original_catalog.incoming_sinks.is_empty()
+        && matches!(operation, AlterTableOperation::DropColumn { .. })
+    {
+        return Err(ErrorCode::InvalidInputSyntax(
+            "dropping columns in target table of sinks is not supported".to_string(),
+        ))?;
     }
 
     match operation {
         AlterTableOperation::AddColumn {
             column_def: new_column,
         } => {
+            fail_if_has_schema_registry()?;
+
             // Duplicated names can actually be checked by `StreamMaterialize`. We do here for
             // better error reporting.
             let new_column_name = new_column.name.real_value();
@@ -171,7 +267,7 @@ pub async fn handle_alter_table_column(
                 ))?
             }
 
-            // Add the new column to the table definition.
+            // Add the new column to the table definition if it is not created by `create table (*)` syntax.
             columns.push(new_column);
         }
 
@@ -184,6 +280,28 @@ pub async fn handle_alter_table_column(
                 bail_not_implemented!(issue = 6903, "drop column cascade");
             }
 
+            // Check if the column to drop is referenced by any generated columns.
+            for column in original_catalog.columns() {
+                if column_name.real_value() == column.name() && !column.is_generated() {
+                    fail_if_has_schema_registry()?;
+                }
+
+                if let Some(expr) = column.generated_expr() {
+                    let expr = ExprImpl::from_expr_proto(expr)?;
+                    let refs = expr.collect_input_refs(original_catalog.columns().len());
+                    for idx in refs.ones() {
+                        let refed_column = &original_catalog.columns()[idx];
+                        if refed_column.name() == column_name.real_value() {
+                            bail!(format!(
+                                "failed to drop column \"{}\" because it's referenced by a generated column \"{}\"",
+                                column_name,
+                                column.name()
+                            ))
+                        }
+                    }
+                }
+            }
+
             // Locate the column by name and remove it.
             let column_name = column_name.real_value();
             let removed_column = columns
@@ -210,7 +328,7 @@ pub async fn handle_alter_table_column(
         }
 
         _ => unreachable!(),
-    }
+    };
 
     replace_table_with_definition(
         &session,
diff --git a/src/frontend/src/handler/alter_table_with_sr.rs b/src/frontend/src/handler/alter_table_with_sr.rs
index c1700c36a3c90..d932246759e22 100644
--- a/src/frontend/src/handler/alter_table_with_sr.rs
+++ b/src/frontend/src/handler/alter_table_with_sr.rs
@@ -12,11 +12,13 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use anyhow::Context;
+use anyhow::{anyhow, Context};
+use fancy_regex::Regex;
 use pgwire::pg_response::StatementType;
 use risingwave_common::bail_not_implemented;
 use risingwave_sqlparser::ast::{ConnectorSchema, ObjectName, Statement};
 use risingwave_sqlparser::parser::Parser;
+use thiserror_ext::AsReport;
 
 use super::alter_source_with_sr::alter_definition_format_encode;
 use super::alter_table_column::{
@@ -24,7 +26,7 @@ use super::alter_table_column::{
 };
 use super::util::SourceSchemaCompatExt;
 use super::{HandlerArgs, RwPgResponse};
-use crate::error::{ErrorCode, Result, RwError};
+use crate::error::{ErrorCode, Result};
 use crate::TableCatalog;
 
 fn get_connector_schema_from_table(table: &TableCatalog) -> Result<Option<ConnectorSchema>> {
@@ -49,13 +51,6 @@ pub async fn handle_refresh_schema(
         bail_not_implemented!("alter table with incoming sinks");
     }
 
-    // TODO(yuhao): alter table with generated columns.
-    if original_table.has_generated_column() {
-        return Err(RwError::from(ErrorCode::BindError(
-            "Alter a table with generated column has not been implemented.".to_string(),
-        )));
-    }
-
     let connector_schema = {
         let connector_schema = get_connector_schema_from_table(&original_table)?;
         if !connector_schema
@@ -81,14 +76,31 @@ pub async fn handle_refresh_schema(
         .try_into()
         .unwrap();
 
-    replace_table_with_definition(
+    let result = replace_table_with_definition(
         &session,
         table_name,
         definition,
         &original_table,
         Some(connector_schema),
     )
-    .await?;
+    .await;
 
-    Ok(RwPgResponse::empty_result(StatementType::ALTER_TABLE))
+    match result {
+        Ok(_) => Ok(RwPgResponse::empty_result(StatementType::ALTER_TABLE)),
+        Err(e) => {
+            let report = e.to_report_string();
+            // This is a workaround for reporting errors when columns to drop is referenced by generated column.
+            // Finding the actual columns to drop requires generating `PbSource` from the sql definition
+            // and fetching schema from schema registry, which will cause a lot of unnecessary refactor.
+            // Here we match the error message to yield when failing to bind generated column exprs.
+            let re = Regex::new(r#"fail to bind expression in generated column "(.*?)""#).unwrap();
+            let captures = re.captures(&report).map_err(anyhow::Error::from)?;
+            if let Some(gen_col_name) = captures.and_then(|captures| captures.get(1)) {
+                Err(anyhow!(e).context(format!("failed to refresh schema because some of the columns to drop are referenced by a generated column \"{}\"",
+                    gen_col_name.as_str())).into())
+            } else {
+                Err(e)
+            }
+        }
+    }
 }
diff --git a/src/frontend/src/handler/create_mv.rs b/src/frontend/src/handler/create_mv.rs
index ed316231e2ce3..0eabe34696150 100644
--- a/src/frontend/src/handler/create_mv.rs
+++ b/src/frontend/src/handler/create_mv.rs
@@ -12,10 +12,13 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::collections::HashSet;
+
 use either::Either;
 use itertools::Itertools;
 use pgwire::pg_response::{PgResponse, StatementType};
 use risingwave_common::acl::AclMode;
+use risingwave_common::catalog::TableId;
 use risingwave_pb::catalog::PbTable;
 use risingwave_pb::stream_plan::stream_fragment_graph::Parallelism;
 use risingwave_sqlparser::ast::{EmitMode, Ident, ObjectName, Query};
@@ -79,7 +82,7 @@ pub(super) fn get_column_names(
     Ok(col_names)
 }
 
-/// Generate create MV plan, return plan and mv table info.
+/// Bind and generate create MV plan, return plan and mv table info.
 pub fn gen_create_mv_plan(
     session: &SessionImpl,
     context: OptimizerContextRef,
@@ -87,6 +90,29 @@ pub fn gen_create_mv_plan(
     name: ObjectName,
     columns: Vec<Ident>,
     emit_mode: Option<EmitMode>,
+) -> Result<(PlanRef, PbTable)> {
+    let mut binder = Binder::new_for_stream(session);
+    let bound = binder.bind_query(query)?;
+    gen_create_mv_plan_bound(
+        session,
+        context,
+        bound,
+        binder.included_relations(),
+        name,
+        columns,
+        emit_mode,
+    )
+}
+
+/// Generate create MV plan from a bound query
+pub fn gen_create_mv_plan_bound(
+    session: &SessionImpl,
+    context: OptimizerContextRef,
+    query: BoundQuery,
+    dependent_relations: HashSet<TableId>,
+    name: ObjectName,
+    columns: Vec<Ident>,
+    emit_mode: Option<EmitMode>,
 ) -> Result<(PlanRef, PbTable)> {
     if session.config().create_compaction_group_for_mv() {
         context.warn_to_user("The session variable CREATE_COMPACTION_GROUP_FOR_MV has been deprecated. It will not take effect.");
@@ -99,23 +125,17 @@ pub fn gen_create_mv_plan(
 
     let definition = context.normalized_sql().to_owned();
 
-    let (dependent_relations, bound) = {
-        let mut binder = Binder::new_for_stream(session);
-        let bound = binder.bind_query(query)?;
-        (binder.included_relations(), bound)
-    };
-
-    let check_items = resolve_query_privileges(&bound);
+    let check_items = resolve_query_privileges(&query);
     session.check_privileges(&check_items)?;
 
-    let col_names = get_column_names(&bound, session, columns)?;
+    let col_names = get_column_names(&query, session, columns)?;
 
     let emit_on_window_close = emit_mode == Some(EmitMode::OnWindowClose);
     if emit_on_window_close {
         context.warn_to_user("EMIT ON WINDOW CLOSE is currently an experimental feature. Please use it with caution.");
     }
 
-    let mut plan_root = Planner::new(context).plan_query(bound)?;
+    let mut plan_root = Planner::new(context).plan_query(query)?;
     if let Some(col_names) = col_names {
         for name in &col_names {
             check_valid_column_name(name)?;
@@ -155,6 +175,32 @@ pub async fn handle_create_mv(
     query: Query,
     columns: Vec<Ident>,
     emit_mode: Option<EmitMode>,
+) -> Result<RwPgResponse> {
+    let (dependent_relations, bound) = {
+        let mut binder = Binder::new_for_stream(handler_args.session.as_ref());
+        let bound = binder.bind_query(query)?;
+        (binder.included_relations(), bound)
+    };
+    handle_create_mv_bound(
+        handler_args,
+        if_not_exists,
+        name,
+        bound,
+        dependent_relations,
+        columns,
+        emit_mode,
+    )
+    .await
+}
+
+pub async fn handle_create_mv_bound(
+    handler_args: HandlerArgs,
+    if_not_exists: bool,
+    name: ObjectName,
+    query: BoundQuery,
+    dependent_relations: HashSet<TableId>,
+    columns: Vec<Ident>,
+    emit_mode: Option<EmitMode>,
 ) -> Result<RwPgResponse> {
     let session = handler_args.session.clone();
 
@@ -176,15 +222,22 @@ pub async fn handle_create_mv(
             ))));
         }
 
-        let has_order_by = !query.order_by.is_empty();
+        let has_order_by = !query.order.is_empty();
         if has_order_by {
             context.warn_to_user(r#"The ORDER BY clause in the CREATE MATERIALIZED VIEW statement does not guarantee that the rows selected out of this materialized view is returned in this order.
 It only indicates the physical clustering of the data, which may improve the performance of queries issued against this materialized view.
 "#.to_string());
         }
 
-        let (plan, table) =
-            gen_create_mv_plan(&session, context.into(), query, name, columns, emit_mode)?;
+        let (plan, table) = gen_create_mv_plan_bound(
+            &session,
+            context.into(),
+            query,
+            dependent_relations,
+            name,
+            columns,
+            emit_mode,
+        )?;
 
         let context = plan.plan_base().ctx().clone();
         let mut graph = build_graph(plan)?;
diff --git a/src/frontend/src/handler/create_secret.rs b/src/frontend/src/handler/create_secret.rs
index 2e99f26e97cb8..9810751361be3 100644
--- a/src/frontend/src/handler/create_secret.rs
+++ b/src/frontend/src/handler/create_secret.rs
@@ -15,6 +15,7 @@
 use pgwire::pg_response::{PgResponse, StatementType};
 use prost::Message;
 use risingwave_common::bail_not_implemented;
+use risingwave_common::license::Feature;
 use risingwave_sqlparser::ast::{CreateSecretStatement, SqlOption, Value};
 
 use crate::error::{ErrorCode, Result};
@@ -30,6 +31,10 @@ pub async fn handle_create_secret(
     handler_args: HandlerArgs,
     stmt: CreateSecretStatement,
 ) -> Result<RwPgResponse> {
+    Feature::SecretManagement
+        .check_available()
+        .map_err(|e| anyhow::anyhow!(e))?;
+
     let session = handler_args.session.clone();
     let db_name = session.database();
     let (schema_name, connection_name) =
@@ -45,15 +50,17 @@ pub async fn handle_create_secret(
         };
     }
 
+    let secret = secret_to_str(&stmt.credential)?.as_bytes().to_vec();
+
     // check if the secret backend is supported
     let with_props = WithOptions::try_from(stmt.with_properties.0.as_ref() as &[SqlOption])?;
     let secret_payload: Vec<u8> = {
-        if let Some(backend) = with_props.inner().get(SECRET_BACKEND_KEY) {
+        if let Some(backend) = with_props.get(SECRET_BACKEND_KEY) {
             match backend.to_lowercase().as_ref() {
                 SECRET_BACKEND_META => {
                     let backend = risingwave_pb::secret::Secret {
                         secret_backend: Some(risingwave_pb::secret::secret::SecretBackend::Meta(
-                            risingwave_pb::secret::SecretMetaBackend { value: vec![] },
+                            risingwave_pb::secret::SecretMetaBackend { value: secret },
                         )),
                     };
                     backend.encode_to_vec()
@@ -100,3 +107,13 @@ pub async fn handle_create_secret(
 
     Ok(PgResponse::empty_result(StatementType::CREATE_SECRET))
 }
+
+fn secret_to_str(value: &Value) -> Result<String> {
+    match value {
+        Value::DoubleQuotedString(s) | Value::SingleQuotedString(s) => Ok(s.to_string()),
+        _ => Err(ErrorCode::InvalidInputSyntax(
+            "secret value should be quoted by ' or \" ".to_string(),
+        )
+        .into()),
+    }
+}
diff --git a/src/frontend/src/handler/create_sink.rs b/src/frontend/src/handler/create_sink.rs
index dd3f435885c28..fdf6a65c4437c 100644
--- a/src/frontend/src/handler/create_sink.rs
+++ b/src/frontend/src/handler/create_sink.rs
@@ -12,8 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::{BTreeMap, BTreeSet, HashMap};
-use std::rc::Rc;
+use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
 use std::sync::{Arc, LazyLock};
 
 use anyhow::Context;
@@ -23,7 +22,10 @@ use itertools::Itertools;
 use maplit::{convert_args, hashmap};
 use pgwire::pg_response::{PgResponse, StatementType};
 use risingwave_common::array::arrow::IcebergArrowConvert;
-use risingwave_common::catalog::{ConnectionId, DatabaseId, Schema, SchemaId, TableId, UserId};
+use risingwave_common::catalog::{
+    ColumnCatalog, ConnectionId, DatabaseId, Schema, SchemaId, TableId, UserId,
+};
+use risingwave_common::secret::LocalSecretManager;
 use risingwave_common::types::DataType;
 use risingwave_common::{bail, catalog};
 use risingwave_connector::sink::catalog::{SinkCatalog, SinkFormatDesc, SinkType};
@@ -31,13 +33,14 @@ use risingwave_connector::sink::iceberg::{IcebergConfig, ICEBERG_SINK};
 use risingwave_connector::sink::{
     CONNECTOR_TYPE_KEY, SINK_TYPE_OPTION, SINK_USER_FORCE_APPEND_ONLY_OPTION, SINK_WITHOUT_BACKFILL,
 };
-use risingwave_pb::catalog::{PbSource, Table};
-use risingwave_pb::ddl_service::ReplaceTablePlan;
+use risingwave_pb::catalog::{PbSink, PbSource, Table};
+use risingwave_pb::ddl_service::{ReplaceTablePlan, TableJobType};
 use risingwave_pb::stream_plan::stream_fragment_graph::Parallelism;
-use risingwave_pb::stream_plan::stream_node::NodeBody;
-use risingwave_pb::stream_plan::{DispatcherType, MergeNode, StreamFragmentGraph, StreamNode};
+use risingwave_pb::stream_plan::stream_node::{NodeBody, PbNodeBody};
+use risingwave_pb::stream_plan::{MergeNode, StreamFragmentGraph, StreamNode};
 use risingwave_sqlparser::ast::{
-    ConnectorSchema, CreateSink, CreateSinkStatement, EmitMode, Encode, Format, Query, Statement,
+    ConnectorSchema, CreateSink, CreateSinkStatement, EmitMode, Encode, ExplainOptions, Format,
+    Query, Statement,
 };
 use risingwave_sqlparser::parser::Parser;
 
@@ -49,6 +52,7 @@ use crate::binder::Binder;
 use crate::catalog::catalog_service::CatalogReadGuard;
 use crate::catalog::source_catalog::SourceCatalog;
 use crate::catalog::view_catalog::ViewCatalog;
+use crate::catalog::SinkId;
 use crate::error::{ErrorCode, Result, RwError};
 use crate::expr::{rewrite_now_to_proctime, ExprImpl, InputRef};
 use crate::handler::alter_table_column::fetch_table_catalog_for_alter;
@@ -60,12 +64,12 @@ use crate::handler::HandlerArgs;
 use crate::optimizer::plan_node::{
     generic, IcebergPartitionInfo, LogicalSource, PartitionComputeInfo, StreamProject,
 };
-use crate::optimizer::{OptimizerContext, OptimizerContextRef, PlanRef, RelationCollectorVisitor};
+use crate::optimizer::{OptimizerContext, PlanRef, RelationCollectorVisitor};
 use crate::scheduler::streaming_manager::CreatingStreamingJobInfo;
 use crate::session::SessionImpl;
 use crate::stream_fragmenter::build_graph;
-use crate::utils::{resolve_privatelink_in_with_option, resolve_secret_in_with_options};
-use crate::{Explain, Planner, TableCatalog, WithOptions};
+use crate::utils::{resolve_privatelink_in_with_option, resolve_secret_ref_in_with_options};
+use crate::{Explain, Planner, TableCatalog, WithOptions, WithOptionsSecResolved};
 
 // used to store result of `gen_sink_plan`
 pub struct SinkPlanContext {
@@ -75,17 +79,36 @@ pub struct SinkPlanContext {
     pub target_table_catalog: Option<Arc<TableCatalog>>,
 }
 
-pub fn gen_sink_plan(
-    session: &SessionImpl,
-    context: OptimizerContextRef,
+pub async fn gen_sink_plan(
+    handler_args: HandlerArgs,
     stmt: CreateSinkStatement,
-    partition_info: Option<PartitionComputeInfo>,
+    explain_options: Option<ExplainOptions>,
 ) -> Result<SinkPlanContext> {
+    let session = handler_args.session.clone();
+    let session = session.as_ref();
     let user_specified_columns = !stmt.columns.is_empty();
     let db_name = session.database();
     let (sink_schema_name, sink_table_name) =
         Binder::resolve_schema_qualified_name(db_name, stmt.sink_name.clone())?;
 
+    let mut with_options = handler_args.with_options.clone();
+
+    let connection_id = {
+        let conn_id =
+            resolve_privatelink_in_with_option(&mut with_options, &sink_schema_name, session)?;
+        conn_id.map(ConnectionId)
+    };
+
+    let mut resolved_with_options = resolve_secret_ref_in_with_options(with_options, session)?;
+
+    let partition_info = get_partition_compute_info(&resolved_with_options).await?;
+
+    let context = if let Some(explain_options) = explain_options {
+        OptimizerContext::new(handler_args.clone(), explain_options)
+    } else {
+        OptimizerContext::from_handler_args(handler_args.clone())
+    };
+
     // Used for debezium's table name
     let sink_from_table_name;
     // `true` means that sink statement has the form: `CREATE SINK s1 FROM ...`
@@ -109,8 +132,6 @@ pub fn gen_sink_plan(
     let (sink_database_id, sink_schema_id) =
         session.get_database_and_schema_id_for_create(sink_schema_name.clone())?;
 
-    let definition = context.normalized_sql().to_owned();
-
     let (dependent_relations, bound) = {
         let mut binder = Binder::new_for_stream(session);
         let bound = binder.bind_query(*query.clone())?;
@@ -127,12 +148,9 @@ pub fn gen_sink_plan(
         get_column_names(&bound, session, stmt.columns)?
     };
 
-    let mut with_options = context.with_options().clone();
-
     if sink_into_table_name.is_some() {
-        let prev = with_options
-            .inner_mut()
-            .insert(CONNECTOR_TYPE_KEY.to_string(), "table".to_string());
+        let prev =
+            resolved_with_options.insert(CONNECTOR_TYPE_KEY.to_string(), "table".to_string());
 
         if prev.is_some() {
             return Err(RwError::from(ErrorCode::BindError(
@@ -141,19 +159,12 @@ pub fn gen_sink_plan(
         }
     }
 
-    let connection_id = {
-        let conn_id =
-            resolve_privatelink_in_with_option(&mut with_options, &sink_schema_name, session)?;
-        conn_id.map(ConnectionId)
-    };
-    let secret_ref = resolve_secret_in_with_options(&mut with_options, session)?;
-
     let emit_on_window_close = stmt.emit_mode == Some(EmitMode::OnWindowClose);
     if emit_on_window_close {
         context.warn_to_user("EMIT ON WINDOW CLOSE is currently an experimental feature. Please use it with caution.");
     }
 
-    let connector = with_options
+    let connector = resolved_with_options
         .get(CONNECTOR_TYPE_KEY)
         .cloned()
         .ok_or_else(|| ErrorCode::BindError(format!("missing field '{CONNECTOR_TYPE_KEY}'")))?;
@@ -162,13 +173,13 @@ pub fn gen_sink_plan(
         // Case A: new syntax `format ... encode ...`
         Some(f) => {
             validate_compatibility(&connector, &f)?;
-            Some(bind_sink_format_desc(f)?)
+            Some(bind_sink_format_desc(session,f)?)
         }
-        None => match with_options.get(SINK_TYPE_OPTION) {
+        None => match resolved_with_options.get(SINK_TYPE_OPTION) {
             // Case B: old syntax `type = '...'`
             Some(t) => SinkFormatDesc::from_legacy_type(&connector, t)?.map(|mut f| {
                 session.notice_to_user("Consider using the newer syntax `FORMAT ... ENCODE ...` instead of `type = '...'`.");
-                if let Some(v) = with_options.get(SINK_USER_FORCE_APPEND_ONLY_OPTION) {
+                if let Some(v) = resolved_with_options.get(SINK_USER_FORCE_APPEND_ONLY_OPTION) {
                     f.options.insert(SINK_USER_FORCE_APPEND_ONLY_OPTION.into(), v.into());
                 }
                 f
@@ -178,12 +189,13 @@ pub fn gen_sink_plan(
         },
     };
 
-    let mut plan_root = Planner::new(context).plan_query(bound)?;
+    let definition = context.normalized_sql().to_owned();
+    let mut plan_root = Planner::new(context.into()).plan_query(bound)?;
     if let Some(col_names) = &col_names {
         plan_root.set_out_names(col_names.clone())?;
     };
 
-    let without_backfill = match with_options.remove(SINK_WITHOUT_BACKFILL) {
+    let without_backfill = match resolved_with_options.remove(SINK_WITHOUT_BACKFILL) {
         Some(flag) if flag.eq_ignore_ascii_case("false") => {
             if direct_sink {
                 true
@@ -227,7 +239,7 @@ pub fn gen_sink_plan(
     let sink_plan = plan_root.gen_sink_plan(
         sink_table_name,
         definition,
-        with_options,
+        resolved_with_options,
         emit_on_window_close,
         db_name.to_owned(),
         sink_from_table_name,
@@ -236,6 +248,7 @@ pub fn gen_sink_plan(
         target_table,
         partition_info,
     )?;
+
     let sink_desc = sink_plan.sink_desc().clone();
 
     let mut sink_plan: PlanRef = sink_plan.into();
@@ -256,7 +269,6 @@ pub fn gen_sink_plan(
         UserId::new(session.user_id()),
         connection_id,
         dependent_relations.into_iter().collect_vec(),
-        secret_ref,
     );
 
     if let Some(table_catalog) = &target_table_catalog {
@@ -281,7 +293,7 @@ pub fn gen_sink_plan(
         let exprs = derive_default_column_project_for_sink(
             &sink_catalog,
             sink_plan.schema(),
-            table_catalog,
+            table_catalog.columns(),
             user_specified_columns,
         )?;
 
@@ -291,6 +303,7 @@ pub fn gen_sink_plan(
 
         let exprs =
             LogicalSource::derive_output_exprs_from_generated_columns(table_catalog.columns())?;
+
         if let Some(exprs) = exprs {
             let logical_project = generic::Project::new(exprs, sink_plan);
             sink_plan = StreamProject::new(logical_project).into();
@@ -310,12 +323,13 @@ pub fn gen_sink_plan(
 // `Some(PartitionComputeInfo)` if the sink need to compute partition.
 // `None` if the sink does not need to compute partition.
 pub async fn get_partition_compute_info(
-    with_options: &WithOptions,
+    with_options: &WithOptionsSecResolved,
 ) -> Result<Option<PartitionComputeInfo>> {
-    let properties = with_options.clone().into_inner();
-    let Some(connector) = properties.get(UPSTREAM_SOURCE_KEY) else {
+    let (options, secret_refs) = with_options.clone().into_parts();
+    let Some(connector) = options.get(UPSTREAM_SOURCE_KEY).cloned() else {
         return Ok(None);
     };
+    let properties = LocalSecretManager::global().fill_secrets(options, secret_refs)?;
     match connector.as_str() {
         ICEBERG_SINK => {
             let iceberg_config = IcebergConfig::from_btreemap(properties)?;
@@ -409,21 +423,17 @@ pub async fn handle_create_sink(
         return Ok(resp);
     }
 
-    let partition_info = get_partition_compute_info(&handle_args.with_options).await?;
-
-    let (sink, graph, target_table_catalog) = {
-        let context = Rc::new(OptimizerContext::from_handler_args(handle_args));
-
+    let (mut sink, graph, target_table_catalog) = {
         let SinkPlanContext {
             query,
             sink_plan: plan,
             sink_catalog: sink,
             target_table_catalog,
-        } = gen_sink_plan(&session, context.clone(), stmt, partition_info)?;
+        } = gen_sink_plan(handle_args, stmt, None).await?;
 
         let has_order_by = !query.order_by.is_empty();
         if has_order_by {
-            context.warn_to_user(
+            plan.ctx().warn_to_user(
                 r#"The ORDER BY clause in the CREATE SINK statement has no effect at all."#
                     .to_string(),
             );
@@ -444,28 +454,44 @@ pub async fn handle_create_sink(
 
     let mut target_table_replace_plan = None;
     if let Some(table_catalog) = target_table_catalog {
+        use crate::handler::alter_table_column::hijack_merger_for_target_table;
+
         check_cycle_for_sink(session.as_ref(), sink.clone(), table_catalog.id())?;
 
         let (mut graph, mut table, source) =
             reparse_table_for_sink(&session, &table_catalog).await?;
 
+        sink.original_target_columns = table
+            .columns
+            .iter()
+            .map(|col| ColumnCatalog::from(col.clone()))
+            .collect_vec();
+
         table
             .incoming_sinks
             .clone_from(&table_catalog.incoming_sinks);
 
-        for _ in 0..(table_catalog.incoming_sinks.len() + 1) {
-            for fragment in graph.fragments.values_mut() {
-                if let Some(node) = &mut fragment.node {
-                    insert_merger_to_union(node);
-                }
-            }
+        let incoming_sink_ids: HashSet<_> = table_catalog.incoming_sinks.iter().copied().collect();
+        let incoming_sinks = fetch_incoming_sinks(&session, &incoming_sink_ids)?;
+
+        for existing_sink in incoming_sinks {
+            hijack_merger_for_target_table(
+                &mut graph,
+                table_catalog.columns(),
+                &existing_sink,
+                Some(&existing_sink.unique_identity()),
+            )?;
         }
 
+        // for new creating sink, we don't have a unique identity because the sink id is not generated yet.
+        hijack_merger_for_target_table(&mut graph, table_catalog.columns(), &sink, None)?;
+
         target_table_replace_plan = Some(ReplaceTablePlan {
             source,
             table: Some(table),
             fragment_graph: Some(graph),
             table_col_index_mapping: None,
+            job_type: TableJobType::General as _,
         });
     }
 
@@ -488,6 +514,24 @@ pub async fn handle_create_sink(
     Ok(PgResponse::empty_result(StatementType::CREATE_SINK))
 }
 
+pub fn fetch_incoming_sinks(
+    session: &Arc<SessionImpl>,
+    incoming_sink_ids: &HashSet<SinkId>,
+) -> Result<Vec<Arc<SinkCatalog>>> {
+    let reader = session.env().catalog_reader().read_guard();
+    let mut sinks = Vec::with_capacity(incoming_sink_ids.len());
+    let db_name = session.database();
+    for schema in reader.iter_schemas(db_name)? {
+        for sink in schema.iter_sink() {
+            if incoming_sink_ids.contains(&sink.id.sink_id) {
+                sinks.push(sink.clone());
+            }
+        }
+    }
+
+    Ok(sinks)
+}
+
 fn check_cycle_for_sink(
     session: &SessionImpl,
     sink_catalog: SinkCatalog,
@@ -634,7 +678,7 @@ pub(crate) async fn reparse_table_for_sink(
         panic!("unexpected statement type: {:?}", definition);
     };
 
-    let (graph, table, source) = generate_stream_graph_for_table(
+    let (graph, table, source, _) = generate_stream_graph_for_table(
         session,
         table_name,
         table_catalog,
@@ -648,21 +692,32 @@ pub(crate) async fn reparse_table_for_sink(
         append_only,
         on_conflict,
         with_version_column,
+        None,
     )
     .await?;
 
     Ok((graph, table, source))
 }
 
-pub(crate) fn insert_merger_to_union(node: &mut StreamNode) {
+pub(crate) fn insert_merger_to_union_with_project(
+    node: &mut StreamNode,
+    project_node: &PbNodeBody,
+    uniq_identity: Option<&str>,
+) {
     if let Some(NodeBody::Union(_union_node)) = &mut node.node_body {
+        // TODO: MergeNode is used as a placeholder, see issue #17658
         node.input.push(StreamNode {
-            identity: "Merge (sink into table)".to_string(),
-            fields: node.fields.clone(),
-            node_body: Some(NodeBody::Merge(MergeNode {
-                upstream_dispatcher_type: DispatcherType::Hash as _,
+            input: vec![StreamNode {
+                node_body: Some(NodeBody::Merge(MergeNode {
+                    ..Default::default()
+                })),
                 ..Default::default()
-            })),
+            }],
+            identity: uniq_identity
+                .unwrap_or(PbSink::UNIQUE_IDENTITY_FOR_CREATING_TABLE_SINK)
+                .to_string(),
+            fields: node.fields.clone(),
+            node_body: Some(project_node.clone()),
             ..Default::default()
         });
 
@@ -670,7 +725,7 @@ pub(crate) fn insert_merger_to_union(node: &mut StreamNode) {
     }
 
     for input in &mut node.input {
-        insert_merger_to_union(input);
+        insert_merger_to_union_with_project(input, project_node, uniq_identity);
     }
 }
 
@@ -683,9 +738,10 @@ fn derive_sink_to_table_expr(
 
     if target_type != input_type {
         bail!(
-            "column type mismatch: {:?} vs {:?}",
+            "column type mismatch: {:?} vs {:?}, column name: {:?}",
             target_type,
-            input_type
+            input_type,
+            sink_schema.fields()[idx].name
         );
     } else {
         Ok(ExprImpl::InputRef(Box::new(InputRef::new(
@@ -695,14 +751,16 @@ fn derive_sink_to_table_expr(
     }
 }
 
-fn derive_default_column_project_for_sink(
+pub(crate) fn derive_default_column_project_for_sink(
     sink: &SinkCatalog,
     sink_schema: &Schema,
-    target_table_catalog: &Arc<TableCatalog>,
+    columns: &[ColumnCatalog],
     user_specified_columns: bool,
 ) -> Result<Vec<ExprImpl>> {
     assert_eq!(sink.full_schema().len(), sink_schema.len());
 
+    let default_column_exprs = TableCatalog::default_column_exprs(columns);
+
     let mut exprs = vec![];
 
     let sink_visible_col_idxes = sink
@@ -718,17 +776,16 @@ fn derive_default_column_project_for_sink(
         .map(|(i, c)| (c.name(), i))
         .collect::<BTreeMap<_, _>>();
 
-    for (idx, table_column) in target_table_catalog.columns().iter().enumerate() {
-        if table_column.is_generated() {
+    for (idx, column) in columns.iter().enumerate() {
+        if column.is_generated() {
             continue;
         }
 
-        let default_col_expr = || -> ExprImpl {
-            rewrite_now_to_proctime(target_table_catalog.default_column_expr(idx))
-        };
+        let default_col_expr =
+            || -> ExprImpl { rewrite_now_to_proctime(default_column_exprs[idx].clone()) };
 
         let sink_col_expr = |sink_col_idx: usize| -> Result<ExprImpl> {
-            derive_sink_to_table_expr(sink_schema, sink_col_idx, table_column.data_type())
+            derive_sink_to_table_expr(sink_schema, sink_col_idx, column.data_type())
         };
 
         // If users specified the columns to be inserted e.g. `CREATE SINK s INTO t(a, b)`, the expressions of `Project` will be generated accordingly.
@@ -736,7 +793,7 @@ fn derive_default_column_project_for_sink(
         // Otherwise, e.g. `CREATE SINK s INTO t`, the columns will be matched by their order in `select` query and the target table.
         #[allow(clippy::collapsible_else_if)]
         if user_specified_columns {
-            if let Some(idx) = sink_visible_col_idxes_by_name.get(table_column.name()) {
+            if let Some(idx) = sink_visible_col_idxes_by_name.get(column.name()) {
                 exprs.push(sink_col_expr(*idx)?);
             } else {
                 exprs.push(default_col_expr());
@@ -755,7 +812,7 @@ fn derive_default_column_project_for_sink(
 /// Transforms the (format, encode, options) from sqlparser AST into an internal struct `SinkFormatDesc`.
 /// This is an analogy to (part of) [`crate::handler::create_source::bind_columns_from_source`]
 /// which transforms sqlparser AST `SourceSchemaV2` into `StreamSourceInfo`.
-fn bind_sink_format_desc(value: ConnectorSchema) -> Result<SinkFormatDesc> {
+fn bind_sink_format_desc(session: &SessionImpl, value: ConnectorSchema) -> Result<SinkFormatDesc> {
     use risingwave_connector::sink::catalog::{SinkEncode, SinkFormat};
     use risingwave_connector::sink::encoder::TimestamptzHandlingMode;
     use risingwave_sqlparser::ast::{Encode as E, Format as F};
@@ -773,7 +830,7 @@ fn bind_sink_format_desc(value: ConnectorSchema) -> Result<SinkFormatDesc> {
         E::Protobuf => SinkEncode::Protobuf,
         E::Avro => SinkEncode::Avro,
         E::Template => SinkEncode::Template,
-        e @ (E::Native | E::Csv | E::Bytes | E::None | E::Text) => {
+        e @ (E::Native | E::Csv | E::Bytes | E::None | E::Text | E::Parquet) => {
             return Err(ErrorCode::BindError(format!("sink encode unsupported: {e}")).into());
         }
     };
@@ -790,7 +847,11 @@ fn bind_sink_format_desc(value: ConnectorSchema) -> Result<SinkFormatDesc> {
         }
     }
 
-    let mut options = WithOptions::try_from(value.row_options.as_slice())?.into_inner();
+    let (mut options, secret_refs) = resolve_secret_ref_in_with_options(
+        WithOptions::try_from(value.row_options.as_slice())?,
+        session,
+    )?
+    .into_parts();
 
     options
         .entry(TimestamptzHandlingMode::OPTION_KEY.to_owned())
@@ -800,6 +861,7 @@ fn bind_sink_format_desc(value: ConnectorSchema) -> Result<SinkFormatDesc> {
         format,
         encode,
         options,
+        secret_refs,
         key_encode,
     })
 }
@@ -868,20 +930,6 @@ pub fn validate_compatibility(connector: &str, format_desc: &ConnectorSchema) ->
     Ok(())
 }
 
-/// For `planner_test` crate so that it does not depend directly on `connector` crate just for `SinkFormatDesc`.
-impl TryFrom<&WithOptions> for Option<SinkFormatDesc> {
-    type Error = risingwave_connector::sink::SinkError;
-
-    fn try_from(value: &WithOptions) -> std::result::Result<Self, Self::Error> {
-        let connector = value.get(CONNECTOR_TYPE_KEY);
-        let r#type = value.get(SINK_TYPE_OPTION);
-        match (connector, r#type) {
-            (Some(c), Some(t)) => SinkFormatDesc::from_legacy_type(c, t),
-            _ => Ok(None),
-        }
-    }
-}
-
 #[cfg(test)]
 pub mod tests {
     use risingwave_common::catalog::{DEFAULT_DATABASE_NAME, DEFAULT_SCHEMA_NAME};
diff --git a/src/frontend/src/handler/create_source.rs b/src/frontend/src/handler/create_source.rs
index 89c5afca08a4c..1c9d0b5893421 100644
--- a/src/frontend/src/handler/create_source.rs
+++ b/src/frontend/src/handler/create_source.rs
@@ -27,22 +27,26 @@ use risingwave_common::catalog::{
     debug_assert_column_ids_distinct, ColumnCatalog, ColumnDesc, ColumnId, Schema, TableId,
     INITIAL_SOURCE_VERSION_ID, KAFKA_TIMESTAMP_COLUMN_NAME,
 };
+use risingwave_common::license::Feature;
+use risingwave_common::secret::LocalSecretManager;
 use risingwave_common::types::DataType;
 use risingwave_connector::parser::additional_columns::{
     build_additional_column_desc, get_supported_additional_columns,
 };
 use risingwave_connector::parser::{
     fetch_json_schema_and_map_to_columns, AvroParserConfig, DebeziumAvroParserConfig,
-    ProtobufParserConfig, SpecificParserConfig, TimestamptzHandling, DEBEZIUM_IGNORE_KEY,
+    ProtobufParserConfig, SchemaLocation, SpecificParserConfig, TimestamptzHandling,
+    DEBEZIUM_IGNORE_KEY,
 };
 use risingwave_connector::schema::schema_registry::{
     name_strategy_from_str, SchemaRegistryAuth, SCHEMA_REGISTRY_PASSWORD, SCHEMA_REGISTRY_USERNAME,
 };
+use risingwave_connector::schema::AWS_GLUE_SCHEMA_ARN_KEY;
 use risingwave_connector::sink::iceberg::IcebergConfig;
 use risingwave_connector::source::cdc::{
     CDC_SHARING_MODE_KEY, CDC_SNAPSHOT_BACKFILL, CDC_SNAPSHOT_MODE_KEY, CDC_TRANSACTIONAL_KEY,
     CDC_WAIT_FOR_STREAMING_START_TIMEOUT, CITUS_CDC_CONNECTOR, MONGODB_CDC_CONNECTOR,
-    MYSQL_CDC_CONNECTOR, POSTGRES_CDC_CONNECTOR,
+    MYSQL_CDC_CONNECTOR, POSTGRES_CDC_CONNECTOR, SQL_SERVER_CDC_CONNECTOR,
 };
 use risingwave_connector::source::datagen::DATAGEN_CONNECTOR;
 use risingwave_connector::source::iceberg::ICEBERG_CONNECTOR;
@@ -81,8 +85,10 @@ use crate::handler::HandlerArgs;
 use crate::optimizer::plan_node::generic::SourceNodeKind;
 use crate::optimizer::plan_node::{LogicalSource, ToStream, ToStreamContext};
 use crate::session::SessionImpl;
-use crate::utils::{resolve_privatelink_in_with_option, resolve_secret_in_with_options};
-use crate::{bind_data_type, build_graph, OptimizerContext, WithOptions};
+use crate::utils::{
+    resolve_privatelink_in_with_option, resolve_secret_ref_in_with_options, OverwriteOptions,
+};
+use crate::{bind_data_type, build_graph, OptimizerContext, WithOptions, WithOptionsSecResolved};
 
 pub(crate) const UPSTREAM_SOURCE_KEY: &str = "connector";
 
@@ -143,7 +149,7 @@ fn json_schema_infer_use_schema_registry(schema_config: &Option<(AstString, bool
 /// Map an Avro schema to a relational schema.
 async fn extract_avro_table_schema(
     info: &StreamSourceInfo,
-    with_properties: &BTreeMap<String, String>,
+    with_properties: &WithOptionsSecResolved,
     format_encode_options: &mut BTreeMap<String, String>,
     is_debezium: bool,
 ) -> Result<Vec<ColumnCatalog>> {
@@ -158,7 +164,7 @@ async fn extract_avro_table_schema(
     } else {
         if let risingwave_connector::parser::EncodingProperties::Avro(avro_props) =
             &parser_config.encoding_config
-            && !avro_props.use_schema_registry
+            && matches!(avro_props.schema_location, SchemaLocation::File { .. })
             && !format_encode_options
                 .get("with_deprecated_file_header")
                 .is_some_and(|v| v == "true")
@@ -179,7 +185,7 @@ async fn extract_avro_table_schema(
 
 async fn extract_debezium_avro_table_pk_columns(
     info: &StreamSourceInfo,
-    with_properties: &WithOptions,
+    with_properties: &WithOptionsSecResolved,
 ) -> Result<Vec<String>> {
     let parser_config = SpecificParserConfig::new(info, with_properties)?;
     let conf = DebeziumAvroParserConfig::new(parser_config.encoding_config).await?;
@@ -189,7 +195,7 @@ async fn extract_debezium_avro_table_pk_columns(
 /// Map a protobuf schema to a relational schema.
 async fn extract_protobuf_table_schema(
     schema: &ProtobufSchema,
-    with_properties: &BTreeMap<String, String>,
+    with_properties: &WithOptionsSecResolved,
     format_encode_options: &mut BTreeMap<String, String>,
 ) -> Result<Vec<ColumnCatalog>> {
     let info = StreamSourceInfo {
@@ -297,15 +303,28 @@ fn get_name_strategy_or_default(name_strategy: Option<AstString>) -> Result<Opti
 pub(crate) async fn bind_columns_from_source(
     session: &SessionImpl,
     source_schema: &ConnectorSchema,
-    with_properties: &BTreeMap<String, String>,
+    with_properties: Either<&WithOptions, &WithOptionsSecResolved>,
 ) -> Result<(Option<Vec<ColumnCatalog>>, StreamSourceInfo)> {
     const MESSAGE_NAME_KEY: &str = "message";
     const KEY_MESSAGE_NAME_KEY: &str = "key.message";
     const NAME_STRATEGY_KEY: &str = "schema.registry.name.strategy";
 
-    let is_kafka: bool = with_properties.is_kafka_connector();
-    let format_encode_options = WithOptions::try_from(source_schema.row_options())?.into_inner();
-    let mut format_encode_options_to_consume = format_encode_options.clone();
+    let options_with_secret = match with_properties {
+        Either::Left(options) => resolve_secret_ref_in_with_options(options.clone(), session)?,
+        Either::Right(options_with_secret) => options_with_secret.clone(),
+    };
+
+    let is_kafka: bool = options_with_secret.is_kafka_connector();
+    let (format_encode_options, format_encode_secret_refs) = resolve_secret_ref_in_with_options(
+        WithOptions::try_from(source_schema.row_options())?,
+        session,
+    )?
+    .into_parts();
+    // Need real secret to access the schema registry
+    let mut format_encode_options_to_consume = LocalSecretManager::global().fill_secrets(
+        format_encode_options.clone(),
+        format_encode_secret_refs.clone(),
+    )?;
 
     fn get_key_message_name(options: &mut BTreeMap<String, String>) -> Option<String> {
         consume_string_from_options(options, KEY_MESSAGE_NAME_KEY)
@@ -332,6 +351,7 @@ pub(crate) async fn bind_columns_from_source(
         format: format_to_prost(&source_schema.format) as i32,
         row_encode: row_encode_to_prost(&source_schema.row_encode) as i32,
         format_encode_options,
+        format_encode_secret_refs,
         ..Default::default()
     };
 
@@ -374,45 +394,55 @@ pub(crate) async fn bind_columns_from_source(
             Some(
                 extract_protobuf_table_schema(
                     &protobuf_schema,
-                    with_properties,
+                    &options_with_secret,
                     &mut format_encode_options_to_consume,
                 )
                 .await?,
             )
         }
         (format @ (Format::Plain | Format::Upsert | Format::Debezium), Encode::Avro) => {
-            let (row_schema_location, use_schema_registry) =
-                get_schema_location(&mut format_encode_options_to_consume)?;
+            if format_encode_options_to_consume
+                .remove(AWS_GLUE_SCHEMA_ARN_KEY)
+                .is_none()
+            {
+                // Legacy logic that assumes either `schema.location` or confluent `schema.registry`.
+                // The handling of newly added aws glue is centralized in `connector::parser`.
+                // TODO(xiangjinwu): move these option parsing to `connector::parser` as well.
 
-            if matches!(format, Format::Debezium) && !use_schema_registry {
-                return Err(RwError::from(ProtocolError(
-                    "schema location for DEBEZIUM_AVRO row format is not supported".to_string(),
-                )));
-            }
+                let (row_schema_location, use_schema_registry) =
+                    get_schema_location(&mut format_encode_options_to_consume)?;
 
-            let message_name = try_consume_string_from_options(
-                &mut format_encode_options_to_consume,
-                MESSAGE_NAME_KEY,
-            );
-            let name_strategy = get_sr_name_strategy_check(
-                &mut format_encode_options_to_consume,
-                use_schema_registry,
-            )?;
+                if matches!(format, Format::Debezium) && !use_schema_registry {
+                    return Err(RwError::from(ProtocolError(
+                        "schema location for DEBEZIUM_AVRO row format is not supported".to_string(),
+                    )));
+                }
 
-            stream_source_info.use_schema_registry = use_schema_registry;
-            stream_source_info
-                .row_schema_location
-                .clone_from(&row_schema_location.0);
-            stream_source_info.proto_message_name = message_name.unwrap_or(AstString("".into())).0;
-            stream_source_info.key_message_name =
-                get_key_message_name(&mut format_encode_options_to_consume);
-            stream_source_info.name_strategy =
-                name_strategy.unwrap_or(PbSchemaRegistryNameStrategy::Unspecified as i32);
+                let message_name = try_consume_string_from_options(
+                    &mut format_encode_options_to_consume,
+                    MESSAGE_NAME_KEY,
+                );
+                let name_strategy = get_sr_name_strategy_check(
+                    &mut format_encode_options_to_consume,
+                    use_schema_registry,
+                )?;
+
+                stream_source_info.use_schema_registry = use_schema_registry;
+                stream_source_info
+                    .row_schema_location
+                    .clone_from(&row_schema_location.0);
+                stream_source_info.proto_message_name =
+                    message_name.unwrap_or(AstString("".into())).0;
+                stream_source_info.key_message_name =
+                    get_key_message_name(&mut format_encode_options_to_consume);
+                stream_source_info.name_strategy =
+                    name_strategy.unwrap_or(PbSchemaRegistryNameStrategy::Unspecified as i32);
+            }
 
             Some(
                 extract_avro_table_schema(
                     &stream_source_info,
-                    with_properties,
+                    &options_with_secret,
                     &mut format_encode_options_to_consume,
                     matches!(format, Format::Debezium),
                 )
@@ -442,6 +472,8 @@ pub(crate) async fn bind_columns_from_source(
 
             None
         }
+        // For parquet format, this step is implemented in parquet parser.
+        (Format::Plain, Encode::Parquet) => None,
         (
             Format::Plain | Format::Upsert | Format::Maxwell | Format::Canal | Format::Debezium,
             Encode::Json,
@@ -468,15 +500,15 @@ pub(crate) async fn bind_columns_from_source(
 
             extract_json_table_schema(
                 &schema_config,
-                with_properties,
+                &options_with_secret,
                 &mut format_encode_options_to_consume,
             )
             .await?
         }
         (Format::None, Encode::None) => {
-            if with_properties.is_iceberg_connector() {
+            if options_with_secret.is_iceberg_connector() {
                 Some(
-                    extract_iceberg_columns(with_properties)
+                    extract_iceberg_columns(&options_with_secret)
                         .await
                         .map_err(|err| ProtocolError(err.to_report_string()))?,
                 )
@@ -512,8 +544,17 @@ fn bind_columns_from_source_for_cdc(
     session: &SessionImpl,
     source_schema: &ConnectorSchema,
 ) -> Result<(Option<Vec<ColumnCatalog>>, StreamSourceInfo)> {
-    let format_encode_options = WithOptions::try_from(source_schema.row_options())?.into_inner();
-    let mut format_encode_options_to_consume = format_encode_options.clone();
+    let (format_encode_options, format_encode_secret_refs) = resolve_secret_ref_in_with_options(
+        WithOptions::try_from(source_schema.row_options())?,
+        session,
+    )?
+    .into_parts();
+
+    // Need real secret to access the schema registry
+    let mut format_encode_options_to_consume = LocalSecretManager::global().fill_secrets(
+        format_encode_options.clone(),
+        format_encode_secret_refs.clone(),
+    )?;
 
     match (&source_schema.format, &source_schema.row_encode) {
         (Format::Plain, Encode::Json) => (),
@@ -536,6 +577,7 @@ fn bind_columns_from_source_for_cdc(
         use_schema_registry: json_schema_infer_use_schema_registry(&schema_config),
         cdc_source_job: true,
         is_distributed: false,
+        format_encode_secret_refs,
         ..Default::default()
     };
     if !format_encode_options_to_consume.is_empty() {
@@ -749,7 +791,7 @@ pub(crate) async fn bind_source_pk(
     source_info: &StreamSourceInfo,
     columns: &mut [ColumnCatalog],
     sql_defined_pk_names: Vec<String>,
-    with_properties: &WithOptions,
+    with_properties: &WithOptionsSecResolved,
 ) -> Result<Vec<String>> {
     let sql_defined_pk = !sql_defined_pk_names.is_empty();
     let include_key_column_name: Option<String> = {
@@ -784,7 +826,10 @@ pub(crate) async fn bind_source_pk(
 
         // For all Upsert formats, we only accept one and only key column as primary key.
         // Additional KEY columns must be set in this case and must be primary key.
-        (Format::Upsert, encode @ Encode::Json | encode @ Encode::Avro) => {
+        (
+            Format::Upsert,
+            encode @ Encode::Json | encode @ Encode::Avro | encode @ Encode::Protobuf,
+        ) => {
             if let Some(ref key_column_name) = include_key_column_name
                 && sql_defined_pk
             {
@@ -979,7 +1024,7 @@ static CONNECTORS_COMPATIBLE_FORMATS: LazyLock<HashMap<String, HashMap<Format, V
         convert_args!(hashmap!(
                 KAFKA_CONNECTOR => hashmap!(
                     Format::Plain => vec![Encode::Json, Encode::Protobuf, Encode::Avro, Encode::Bytes, Encode::Csv],
-                    Format::Upsert => vec![Encode::Json, Encode::Avro],
+                    Format::Upsert => vec![Encode::Json, Encode::Avro, Encode::Protobuf],
                     Format::Debezium => vec![Encode::Json, Encode::Avro],
                     Format::Maxwell => vec![Encode::Json],
                     Format::Canal => vec![Encode::Json],
@@ -1017,7 +1062,7 @@ static CONNECTORS_COMPATIBLE_FORMATS: LazyLock<HashMap<String, HashMap<Format, V
                     Format::Plain => vec![Encode::Csv, Encode::Json],
                 ),
                 OPENDAL_S3_CONNECTOR => hashmap!(
-                    Format::Plain => vec![Encode::Csv, Encode::Json],
+                    Format::Plain => vec![Encode::Csv, Encode::Json, Encode::Parquet],
                 ),
                 GCS_CONNECTOR => hashmap!(
                     Format::Plain => vec![Encode::Csv, Encode::Json],
@@ -1052,10 +1097,24 @@ static CONNECTORS_COMPATIBLE_FORMATS: LazyLock<HashMap<String, HashMap<Format, V
                 ),
                 ICEBERG_CONNECTOR => hashmap!(
                     Format::None => vec![Encode::None],
-                )
+                ),
+                SQL_SERVER_CDC_CONNECTOR => hashmap!(
+                    Format::Debezium => vec![Encode::Json],
+                    // support source stream job
+                    Format::Plain => vec![Encode::Json],
+                ),
         ))
     });
 
+pub fn validate_license(connector: &str) -> Result<()> {
+    if connector == SQL_SERVER_CDC_CONNECTOR {
+        Feature::SqlServerCdcSource
+            .check_available()
+            .map_err(|e| anyhow::anyhow!(e))?;
+    }
+    Ok(())
+}
+
 pub fn validate_compatibility(
     source_schema: &ConnectorSchema,
     props: &mut BTreeMap<String, String>,
@@ -1073,6 +1132,8 @@ pub fn validate_compatibility(
                 CONNECTORS_COMPATIBLE_FORMATS.keys()
             )))
         })?;
+
+    validate_license(&connector)?;
     if connector != KAFKA_CONNECTOR {
         let res = match (&source_schema.format, &source_schema.row_encode) {
             (Format::Plain, Encode::Protobuf) | (Format::Plain, Encode::Avro) => {
@@ -1114,12 +1175,30 @@ pub fn validate_compatibility(
     }
 
     if connector == POSTGRES_CDC_CONNECTOR || connector == CITUS_CDC_CONNECTOR {
-        if !props.contains_key("slot.name") {
-            // Build a random slot name with UUID
-            // e.g. "rw_cdc_f9a3567e6dd54bf5900444c8b1c03815"
-            let uuid = uuid::Uuid::new_v4().to_string().replace('-', "");
-            props.insert("slot.name".into(), format!("rw_cdc_{}", uuid));
+        match props.get("slot.name") {
+            None => {
+                // Build a random slot name with UUID
+                // e.g. "rw_cdc_f9a3567e6dd54bf5900444c8b1c03815"
+                let uuid = uuid::Uuid::new_v4();
+                props.insert("slot.name".into(), format!("rw_cdc_{}", uuid.simple()));
+            }
+            Some(slot_name) => {
+                // please refer to
+                // - https://github.com/debezium/debezium/blob/97956ce25b7612e3413d363658661896b7d2e0a2/debezium-connector-postgres/src/main/java/io/debezium/connector/postgresql/PostgresConnectorConfig.java#L1179
+                // - https://doxygen.postgresql.org/slot_8c.html#afac399f07320b9adfd2c599cf822aaa3
+                if !slot_name
+                    .chars()
+                    .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '_')
+                    || slot_name.len() > 63
+                {
+                    return Err(RwError::from(ProtocolError(format!(
+                        "Invalid replication slot name: {:?}. Valid replication slot name must contain only digits, lowercase characters and underscores with length <= 63",
+                        slot_name
+                    ))));
+                }
+            }
         }
+
         if !props.contains_key("schema.name") {
             // Default schema name is "public"
             props.insert("schema.name".into(), "public".into());
@@ -1133,6 +1212,12 @@ pub fn validate_compatibility(
             props.insert("publication.create.enable".into(), "true".into());
         }
     }
+
+    if connector == SQL_SERVER_CDC_CONNECTOR && !props.contains_key("schema.name") {
+        // Default schema name is "dbo"
+        props.insert("schema.name".into(), "dbo".into());
+    }
+
     Ok(())
 }
 
@@ -1142,7 +1227,7 @@ pub fn validate_compatibility(
 /// One should only call this function after all properties of all columns are resolved, like
 /// generated column descriptors.
 pub(super) async fn check_source_schema(
-    props: &WithOptions,
+    props: &WithOptionsSecResolved,
     row_id_index: Option<usize>,
     columns: &[ColumnCatalog],
 ) -> Result<()> {
@@ -1151,9 +1236,9 @@ pub(super) async fn check_source_schema(
     };
 
     if connector == NEXMARK_CONNECTOR {
-        check_nexmark_schema(props.inner(), row_id_index, columns)
+        check_nexmark_schema(props, row_id_index, columns)
     } else if connector == ICEBERG_CONNECTOR {
-        Ok(check_iceberg_source(props.inner(), columns)
+        Ok(check_iceberg_source(props, columns)
             .await
             .map_err(|err| ProtocolError(err.to_report_string()))?)
     } else {
@@ -1162,7 +1247,7 @@ pub(super) async fn check_source_schema(
 }
 
 pub(super) fn check_nexmark_schema(
-    props: &BTreeMap<String, String>,
+    props: &WithOptionsSecResolved,
     row_id_index: Option<usize>,
     columns: &[ColumnCatalog],
 ) -> Result<()> {
@@ -1216,7 +1301,7 @@ pub(super) fn check_nexmark_schema(
 }
 
 pub async fn extract_iceberg_columns(
-    with_properties: &BTreeMap<String, String>,
+    with_properties: &WithOptionsSecResolved,
 ) -> anyhow::Result<Vec<ColumnCatalog>> {
     let props = ConnectorProperties::extract(with_properties.clone(), true)?;
     if let ConnectorProperties::Iceberg(properties) = props {
@@ -1255,7 +1340,7 @@ pub async fn extract_iceberg_columns(
 }
 
 pub async fn check_iceberg_source(
-    props: &BTreeMap<String, String>,
+    props: &WithOptionsSecResolved,
     columns: &[ColumnCatalog],
 ) -> anyhow::Result<()> {
     let props = ConnectorProperties::extract(props.clone(), true)?;
@@ -1314,13 +1399,22 @@ pub fn bind_connector_props(
     let mut with_properties = handler_args.with_options.clone().into_connector_props();
     validate_compatibility(source_schema, &mut with_properties)?;
     let create_cdc_source_job = with_properties.is_shareable_cdc_connector();
+
+    if !is_create_source && with_properties.is_shareable_only_cdc_connector() {
+        return Err(RwError::from(ProtocolError(format!(
+            "connector {} does not support `CREATE TABLE`, please use `CREATE SOURCE` instead",
+            with_properties.get_connector().unwrap(),
+        ))));
+    }
     if is_create_source && create_cdc_source_job {
         // set connector to backfill mode
         with_properties.insert(CDC_SNAPSHOT_MODE_KEY.into(), CDC_SNAPSHOT_BACKFILL.into());
         // enable cdc sharing mode, which will capture all tables in the given `database.name`
         with_properties.insert(CDC_SHARING_MODE_KEY.into(), "true".into());
         // enable transactional cdc
-        with_properties.insert(CDC_TRANSACTIONAL_KEY.into(), "true".into());
+        if with_properties.enable_transaction_metadata() {
+            with_properties.insert(CDC_TRANSACTIONAL_KEY.into(), "true".into());
+        }
         with_properties.insert(
             CDC_WAIT_FOR_STREAMING_START_TIMEOUT.into(),
             handler_args
@@ -1330,7 +1424,7 @@ pub fn bind_connector_props(
                 .to_string(),
         );
     }
-    Ok(WithOptions::new(with_properties))
+    Ok(with_properties)
 }
 
 #[allow(clippy::too_many_arguments)]
@@ -1349,6 +1443,7 @@ pub async fn bind_create_source_or_table_with_connector(
     col_id_gen: &mut ColumnIdGenerator,
     // `true` for "create source", `false` for "create table with connector"
     is_create_source: bool,
+    source_rate_limit: Option<u32>,
 ) -> Result<(SourceCatalog, DatabaseId, SchemaId)> {
     let session = &handler_args.session;
     let db_name: &str = session.database();
@@ -1404,6 +1499,13 @@ pub async fn bind_create_source_or_table_with_connector(
         check_and_add_timestamp_column(&with_properties, &mut columns);
     }
 
+    // resolve privatelink connection for Kafka
+    let mut with_properties = with_properties;
+    let connection_id =
+        resolve_privatelink_in_with_option(&mut with_properties, &schema_name, session)?;
+
+    let with_properties = resolve_secret_ref_in_with_options(with_properties, session)?;
+
     let pk_names = bind_source_pk(
         &source_schema,
         &source_info,
@@ -1456,13 +1558,7 @@ pub async fn bind_create_source_or_table_with_connector(
     )?;
     check_source_schema(&with_properties, row_id_index, &columns).await?;
 
-    // resolve privatelink connection for Kafka
-    let mut with_properties = with_properties;
-    let connection_id =
-        resolve_privatelink_in_with_option(&mut with_properties, &schema_name, session)?;
-    let _secret_ref = resolve_secret_in_with_options(&mut with_properties, session)?;
-
-    let definition: String = handler_args.normalized_sql.clone();
+    let definition = handler_args.normalized_sql.clone();
 
     let associated_table_id = if is_create_source {
         None
@@ -1478,7 +1574,7 @@ pub async fn bind_create_source_or_table_with_connector(
         owner: session.user_id(),
         info: source_info,
         row_id_index,
-        with_properties: with_properties.into_inner().into_iter().collect(),
+        with_properties,
         watermark_descs,
         associated_table_id,
         definition,
@@ -1488,15 +1584,17 @@ pub async fn bind_create_source_or_table_with_connector(
         version: INITIAL_SOURCE_VERSION_ID,
         created_at_cluster_version: None,
         initialized_at_cluster_version: None,
+        rate_limit: source_rate_limit,
     };
     Ok((source, database_id, schema_id))
 }
 
 pub async fn handle_create_source(
-    handler_args: HandlerArgs,
+    mut handler_args: HandlerArgs,
     stmt: CreateSourceStatement,
 ) -> Result<RwPgResponse> {
     let session = handler_args.session.clone();
+    let overwrite_options = OverwriteOptions::new(&mut handler_args);
 
     if let Either::Right(resp) = session.check_relation_name_duplicated(
         stmt.source_name.clone(),
@@ -1522,7 +1620,7 @@ pub async fn handle_create_source(
     let (columns_from_resolve_source, mut source_info) = if create_cdc_source_job {
         bind_columns_from_source_for_cdc(&session, &source_schema)?
     } else {
-        bind_columns_from_source(&session, &source_schema, &with_properties).await?
+        bind_columns_from_source(&session, &source_schema, Either::Left(&with_properties)).await?
     };
     if is_shared {
         // Note: this field should be called is_shared. Check field doc for more details.
@@ -1545,6 +1643,7 @@ pub async fn handle_create_source(
         stmt.include_column_options,
         &mut col_id_gen,
         true,
+        overwrite_options.source_rate_limit,
     )
     .await?;
 
@@ -1605,6 +1704,7 @@ fn row_encode_to_prost(row_encode: &Encode) -> EncodeType {
         Encode::Csv => EncodeType::Csv,
         Encode::Bytes => EncodeType::Bytes,
         Encode::Template => EncodeType::Template,
+        Encode::Parquet => EncodeType::Parquet,
         Encode::None => EncodeType::None,
         Encode::Text => EncodeType::Text,
     }
diff --git a/src/frontend/src/handler/create_table.rs b/src/frontend/src/handler/create_table.rs
index 16d0bb8e88d7c..6e57ab52c87fe 100644
--- a/src/frontend/src/handler/create_table.rs
+++ b/src/frontend/src/handler/create_table.rs
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::{BTreeMap, HashMap};
+use std::collections::HashMap;
 use std::rc::Rc;
 use std::sync::Arc;
 
@@ -26,12 +26,13 @@ use risingwave_common::catalog::{
     CdcTableDesc, ColumnCatalog, ColumnDesc, TableId, TableVersionId, DEFAULT_SCHEMA_NAME,
     INITIAL_TABLE_VERSION_ID,
 };
+use risingwave_common::license::Feature;
 use risingwave_common::util::sort_util::{ColumnOrder, OrderType};
 use risingwave_common::util::value_encoding::DatumToProtoExt;
-use risingwave_connector::source;
 use risingwave_connector::source::cdc::external::{
     ExternalTableConfig, ExternalTableImpl, DATABASE_NAME_KEY, SCHEMA_NAME_KEY, TABLE_NAME_KEY,
 };
+use risingwave_connector::{source, WithOptionsSecResolved};
 use risingwave_pb::catalog::{PbSource, PbTable, Table, WatermarkDesc};
 use risingwave_pb::ddl_service::TableJobType;
 use risingwave_pb::plan_common::column_desc::GeneratedOrDefaultColumn;
@@ -45,6 +46,7 @@ use risingwave_sqlparser::ast::{
     ExplainOptions, Format, ObjectName, OnConflict, SourceWatermark, TableConstraint,
 };
 use risingwave_sqlparser::parser::IncludeOption;
+use thiserror_ext::AsReport;
 
 use super::RwPgResponse;
 use crate::binder::{bind_data_type, bind_struct_field, Clause};
@@ -65,6 +67,7 @@ use crate::optimizer::property::{Order, RequiredDist};
 use crate::optimizer::{OptimizerContext, OptimizerContextRef, PlanRef, PlanRoot};
 use crate::session::SessionImpl;
 use crate::stream_fragmenter::build_graph;
+use crate::utils::OverwriteOptions;
 use crate::{Binder, TableCatalog, WithOptions};
 
 /// Column ID generator for a new table or a new version of an existing table to alter.
@@ -185,7 +188,7 @@ pub fn bind_sql_columns(column_defs: &[ColumnDef]) -> Result<Vec<ColumnCatalog>>
                 _ => {
                     return Err(ErrorCode::NotSupported(
                         format!("{} is not a collatable data type", data_type),
-                        "The only built-in collatable data types are `varchar`, please check your type".into()
+                        "The only built-in collatable data types are `varchar`, please check your type".into(),
                     ).into());
                 }
             }
@@ -288,7 +291,12 @@ pub fn bind_sql_column_constraints(
                     binder.set_clause(Some(Clause::GeneratedColumn));
                     let idx = binder
                         .get_column_binding_index(table_name.clone(), &column.name.real_value())?;
-                    let expr_impl = binder.bind_expr(expr)?;
+                    let expr_impl = binder.bind_expr(expr).with_context(|| {
+                        format!(
+                            "fail to bind expression in generated column \"{}\"",
+                            column.name.real_value()
+                        )
+                    })?;
 
                     check_generated_column_constraints(
                         &column.name.real_value(),
@@ -327,12 +335,12 @@ pub fn bind_sql_column_constraints(
                     if let Some(snapshot_value) = rewritten_expr_impl.try_fold_const() {
                         let snapshot_value = snapshot_value?;
 
-                        column_catalogs[idx].column_desc.generated_or_default_column = Some(
-                            GeneratedOrDefaultColumn::DefaultColumn(DefaultColumnDesc {
+                        column_catalogs[idx].column_desc.generated_or_default_column =
+                            Some(GeneratedOrDefaultColumn::DefaultColumn(DefaultColumnDesc {
                                 snapshot_value: Some(snapshot_value.to_protobuf()),
-                                expr: Some(expr_impl.to_expr_proto()), /* persist the original expression */
-                            }),
-                        );
+                                expr: Some(expr_impl.to_expr_proto()),
+                                // persist the original expression
+                            }));
                     } else {
                         return Err(ErrorCode::BindError(format!(
                             "Default expression used in column `{}` cannot be evaluated. \
@@ -452,7 +460,7 @@ pub fn bind_pk_and_row_id_on_relation(
 /// stream source.
 #[allow(clippy::too_many_arguments)]
 pub(crate) async fn gen_create_table_plan_with_source(
-    handler_args: HandlerArgs,
+    mut handler_args: HandlerArgs,
     explain_options: ExplainOptions,
     table_name: ObjectName,
     column_defs: Vec<ColumnDef>,
@@ -481,8 +489,10 @@ pub(crate) async fn gen_create_table_plan_with_source(
     let with_properties = bind_connector_props(&handler_args, &source_schema, false)?;
 
     let (columns_from_resolve_source, source_info) =
-        bind_columns_from_source(session, &source_schema, &with_properties).await?;
+        bind_columns_from_source(session, &source_schema, Either::Left(&with_properties)).await?;
 
+    let overwrite_options = OverwriteOptions::new(&mut handler_args);
+    let rate_limit = overwrite_options.source_rate_limit;
     let (source_catalog, database_id, schema_id) = bind_create_source_or_table_with_connector(
         handler_args.clone(),
         table_name,
@@ -497,6 +507,7 @@ pub(crate) async fn gen_create_table_plan_with_source(
         include_column_options,
         &mut col_id_gen,
         false,
+        rate_limit,
     )
     .await?;
 
@@ -536,6 +547,12 @@ pub(crate) fn gen_create_table_plan(
     for c in &mut columns {
         c.column_desc.column_id = col_id_gen.generate(c.name())
     }
+
+    let (_, secret_refs) = context.with_options().clone().into_parts();
+    if !secret_refs.is_empty() {
+        return Err(crate::error::ErrorCode::InvalidParameterValue("Secret reference is not allowed in options when creating table without external source".to_string()).into());
+    }
+
     gen_create_table_plan_without_source(
         context,
         table_name,
@@ -660,6 +677,7 @@ fn gen_table_plan_inner(
     let session = context.session_ctx().clone();
     let retention_seconds = context.with_options().retention_seconds();
     let is_external_source = source_catalog.is_some();
+
     let source_node: PlanRef = LogicalSource::new(
         source_catalog.map(|source| Rc::new(source.clone())),
         columns.clone(),
@@ -724,6 +742,9 @@ fn gen_table_plan_inner(
     Ok((materialize.into(), table))
 }
 
+/// Generate stream plan for cdc table based on shared source.
+/// In replace workflow, the `table_id` is the id of the table to be replaced
+/// in create table workflow, the `table_id` is a placeholder will be filled in the Meta
 #[allow(clippy::too_many_arguments)]
 pub(crate) fn gen_create_table_plan_for_cdc_table(
     handler_args: HandlerArgs,
@@ -732,7 +753,7 @@ pub(crate) fn gen_create_table_plan_for_cdc_table(
     external_table_name: String,
     mut columns: Vec<ColumnCatalog>,
     pk_names: Vec<String>,
-    connect_properties: BTreeMap<String, String>,
+    connect_properties: WithOptionsSecResolved,
     mut col_id_gen: ColumnIdGenerator,
     on_conflict: Option<OnConflict>,
     with_version_column: Option<String>,
@@ -740,6 +761,7 @@ pub(crate) fn gen_create_table_plan_for_cdc_table(
     resolved_table_name: String,
     database_id: DatabaseId,
     schema_id: SchemaId,
+    table_id: TableId,
 ) -> Result<(PlanRef, PbTable)> {
     let context: OptimizerContextRef = OptimizerContext::new(handler_args, explain_options).into();
     let session = context.session_ctx().clone();
@@ -777,14 +799,17 @@ pub(crate) fn gen_create_table_plan_for_cdc_table(
         .map(|idx| ColumnOrder::new(*idx, OrderType::ascending()))
         .collect();
 
+    let (options, secret_refs) = connect_properties.into_parts();
+
     let cdc_table_desc = CdcTableDesc {
-        table_id: TableId::placeholder(), // will be filled in meta node
-        source_id: source.id.into(),      // id of cdc source streaming job
+        table_id,
+        source_id: source.id.into(), // id of cdc source streaming job
         external_table_name: external_table_name.clone(),
         pk: table_pk,
         columns: columns.iter().map(|c| c.column_desc.clone()).collect(),
         stream_key: pk_column_indices,
-        connect_properties: connect_properties.into_iter().collect(),
+        connect_properties: options,
+        secret_refs,
     };
 
     tracing::debug!(?cdc_table_desc, "create cdc table");
@@ -832,10 +857,10 @@ pub(crate) fn gen_create_table_plan_for_cdc_table(
 }
 
 fn derive_connect_properties(
-    source_with_properties: &BTreeMap<String, String>,
+    source_with_properties: &WithOptionsSecResolved,
     external_table_name: String,
-) -> Result<BTreeMap<String, String>> {
-    use source::cdc::{MYSQL_CDC_CONNECTOR, POSTGRES_CDC_CONNECTOR};
+) -> Result<WithOptionsSecResolved> {
+    use source::cdc::{MYSQL_CDC_CONNECTOR, POSTGRES_CDC_CONNECTOR, SQL_SERVER_CDC_CONNECTOR};
     // we should remove the prefix from `full_table_name`
     let mut connect_properties = source_with_properties.clone();
     if let Some(connector) = source_with_properties.get(UPSTREAM_SOURCE_KEY) {
@@ -860,6 +885,16 @@ fn derive_connect_properties(
 
                 table_name
             }
+            SQL_SERVER_CDC_CONNECTOR => {
+                let (schema_name, table_name) = external_table_name
+                    .split_once('.')
+                    .ok_or_else(|| anyhow!("The upstream table name must contain schema name prefix, e.g. 'dbo.table'"))?;
+
+                // insert 'schema.name' into connect properties
+                connect_properties.insert(SCHEMA_NAME_KEY.into(), schema_name.into());
+
+                table_name
+            }
             _ => {
                 return Err(RwError::from(anyhow!(
                     "connector {} is not supported for cdc table",
@@ -994,6 +1029,7 @@ pub(super) async fn handle_create_table_plan(
                     resolved_table_name,
                     database_id,
                     schema_id,
+                    TableId::placeholder(),
                 )?;
 
                 ((plan, None, table), TableJobType::SharedCdcSource)
@@ -1081,12 +1117,21 @@ fn sanity_check_for_cdc_table(
 async fn derive_schema_for_cdc_table(
     column_defs: &Vec<ColumnDef>,
     constraints: &Vec<TableConstraint>,
-    connect_properties: BTreeMap<String, String>,
+    connect_properties: WithOptionsSecResolved,
     need_auto_schema_map: bool,
 ) -> Result<(Vec<ColumnCatalog>, Vec<String>)> {
     // read cdc table schema from external db or parsing the schema from SQL definitions
     if need_auto_schema_map {
-        let config = ExternalTableConfig::try_from_btreemap(connect_properties)
+        Feature::CdcTableSchemaMap
+            .check_available()
+            .map_err(|err| {
+                ErrorCode::NotSupported(
+                    err.to_report_string(),
+                    "Please define the schema manually".to_owned(),
+                )
+            })?;
+        let (options, secret_refs) = connect_properties.into_parts();
+        let config = ExternalTableConfig::try_from_btreemap(options, secret_refs)
             .context("failed to extract external table config")?;
 
         let table = ExternalTableImpl::connect(config)
@@ -1195,7 +1240,7 @@ pub fn check_create_table_with_source(
     if cdc_table_info.is_some() {
         return Ok(source_schema);
     }
-    let defined_source = with_options.inner().contains_key(UPSTREAM_SOURCE_KEY);
+    let defined_source = with_options.contains_key(UPSTREAM_SOURCE_KEY);
     if !include_column_options.is_empty() && !defined_source {
         return Err(ErrorCode::InvalidInputSyntax(
             "INCLUDE should be used with a connector".to_owned(),
@@ -1218,23 +1263,24 @@ pub async fn generate_stream_graph_for_table(
     source_schema: Option<ConnectorSchema>,
     handler_args: HandlerArgs,
     col_id_gen: ColumnIdGenerator,
-    columns: Vec<ColumnDef>,
+    column_defs: Vec<ColumnDef>,
     wildcard_idx: Option<usize>,
     constraints: Vec<TableConstraint>,
     source_watermarks: Vec<SourceWatermark>,
     append_only: bool,
     on_conflict: Option<OnConflict>,
     with_version_column: Option<String>,
-) -> Result<(StreamFragmentGraph, Table, Option<PbSource>)> {
+    cdc_table_info: Option<CdcTableInfo>,
+) -> Result<(StreamFragmentGraph, Table, Option<PbSource>, TableJobType)> {
     use risingwave_pb::catalog::table::OptionalAssociatedSourceId;
 
-    let (plan, source, table) = match source_schema {
-        Some(source_schema) => {
+    let ((plan, source, table), job_type) = match (source_schema, cdc_table_info.as_ref()) {
+        (Some(source_schema), None) => (
             gen_create_table_plan_with_source(
                 handler_args,
                 ExplainOptions::default(),
                 table_name,
-                columns,
+                column_defs,
                 wildcard_idx,
                 constraints,
                 source_schema,
@@ -1245,14 +1291,15 @@ pub async fn generate_stream_graph_for_table(
                 with_version_column,
                 vec![],
             )
-            .await?
-        }
-        None => {
+            .await?,
+            TableJobType::General,
+        ),
+        (None, None) => {
             let context = OptimizerContext::from_handler_args(handler_args);
             let (plan, table) = gen_create_table_plan(
                 context,
                 table_name,
-                columns,
+                column_defs,
                 constraints,
                 col_id_gen,
                 source_watermarks,
@@ -1260,7 +1307,53 @@ pub async fn generate_stream_graph_for_table(
                 on_conflict,
                 with_version_column,
             )?;
-            (plan, None, table)
+            ((plan, None, table), TableJobType::General)
+        }
+        (None, Some(cdc_table)) => {
+            let session = &handler_args.session;
+            let (source, resolved_table_name, database_id, schema_id) =
+                get_source_and_resolved_table_name(session, cdc_table.clone(), table_name.clone())?;
+
+            let connect_properties = derive_connect_properties(
+                &source.with_properties,
+                cdc_table.external_table_name.clone(),
+            )?;
+
+            let (columns, pk_names) = derive_schema_for_cdc_table(
+                &column_defs,
+                &constraints,
+                connect_properties.clone(),
+                false,
+            )
+            .await?;
+
+            let (plan, table) = gen_create_table_plan_for_cdc_table(
+                handler_args,
+                ExplainOptions::default(),
+                source,
+                cdc_table.external_table_name.clone(),
+                columns,
+                pk_names,
+                connect_properties,
+                col_id_gen,
+                on_conflict,
+                with_version_column,
+                vec![], // empty include options
+                resolved_table_name,
+                database_id,
+                schema_id,
+                original_catalog.id(),
+            )?;
+
+            ((plan, None, table), TableJobType::SharedCdcSource)
+        }
+        (Some(_), Some(_)) => {
+            return Err(ErrorCode::NotSupported(
+                "Data format and encoding format doesn't apply to table created from a CDC source"
+                    .into(),
+                "Remove the FORMAT and ENCODE specification".into(),
+            )
+            .into());
         }
     };
 
@@ -1290,7 +1383,35 @@ pub async fn generate_stream_graph_for_table(
         ..table
     };
 
-    Ok((graph, table, source))
+    Ok((graph, table, source, job_type))
+}
+
+fn get_source_and_resolved_table_name(
+    session: &Arc<SessionImpl>,
+    cdc_table: CdcTableInfo,
+    table_name: ObjectName,
+) -> Result<(Arc<SourceCatalog>, String, DatabaseId, SchemaId)> {
+    let db_name = session.database();
+    let (schema_name, resolved_table_name) =
+        Binder::resolve_schema_qualified_name(db_name, table_name)?;
+    let (database_id, schema_id) =
+        session.get_database_and_schema_id_for_create(schema_name.clone())?;
+
+    let (source_schema, source_name) =
+        Binder::resolve_schema_qualified_name(db_name, cdc_table.source_name.clone())?;
+
+    let source = {
+        let catalog_reader = session.env().catalog_reader().read_guard();
+        let schema_name = source_schema.unwrap_or(DEFAULT_SCHEMA_NAME.to_string());
+        let (source, _) = catalog_reader.get_source_by_name(
+            db_name,
+            SchemaPath::Name(schema_name.as_str()),
+            source_name.as_str(),
+        )?;
+        source.clone()
+    };
+
+    Ok((source, resolved_table_name, database_id, schema_id))
 }
 
 #[cfg(test)]
diff --git a/src/frontend/src/handler/create_table_as.rs b/src/frontend/src/handler/create_table_as.rs
index 9a01d2919086e..bb00be2dfa486 100644
--- a/src/frontend/src/handler/create_table_as.rs
+++ b/src/frontend/src/handler/create_table_as.rs
@@ -90,6 +90,13 @@ pub async fn handle_create_as(
 
     let (graph, source, table) = {
         let context = OptimizerContext::from_handler_args(handler_args.clone());
+        let (_, secret_refs) = context.with_options().clone().into_parts();
+        if !secret_refs.is_empty() {
+            return Err(crate::error::ErrorCode::InvalidParameterValue(
+                "Secret reference is not allowed in options for CREATE TABLE AS".to_string(),
+            )
+            .into());
+        }
         let (plan, table) = gen_create_table_plan_without_source(
             context,
             table_name.clone(),
diff --git a/src/frontend/src/handler/create_view.rs b/src/frontend/src/handler/create_view.rs
index 673fc149dd8c6..5ad0e8956b967 100644
--- a/src/frontend/src/handler/create_view.rs
+++ b/src/frontend/src/handler/create_view.rs
@@ -87,12 +87,20 @@ pub async fn handle_create_view(
             .collect()
     };
 
+    let (properties, secret_refs) = properties.into_parts();
+    if !secret_refs.is_empty() {
+        return Err(crate::error::ErrorCode::InvalidParameterValue(
+            "Secret reference is not allowed in create view options".to_string(),
+        )
+        .into());
+    }
+
     let view = PbView {
         id: 0,
         schema_id,
         database_id,
         name: view_name,
-        properties: properties.inner().clone().into_iter().collect(),
+        properties,
         owner: session.user_id(),
         dependent_relations: dependent_relations
             .into_iter()
diff --git a/src/frontend/src/handler/drop_secret.rs b/src/frontend/src/handler/drop_secret.rs
index 37fbd2cedd408..eff4b35224b8b 100644
--- a/src/frontend/src/handler/drop_secret.rs
+++ b/src/frontend/src/handler/drop_secret.rs
@@ -13,6 +13,7 @@
 // limitations under the License.
 
 use pgwire::pg_response::StatementType;
+use risingwave_common::license::Feature;
 use risingwave_sqlparser::ast::ObjectName;
 
 use crate::catalog::root_catalog::SchemaPath;
@@ -25,6 +26,10 @@ pub async fn handle_drop_secret(
     secret_name: ObjectName,
     if_exists: bool,
 ) -> Result<RwPgResponse> {
+    Feature::SecretManagement
+        .check_available()
+        .map_err(|e| anyhow::anyhow!(e))?;
+
     let session = handler_args.session;
     let db_name = session.database();
     let (schema_name, secret_name) = Binder::resolve_schema_qualified_name(db_name, secret_name)?;
@@ -52,13 +57,11 @@ pub async fn handle_drop_secret(
             };
         session.check_privilege_for_drop_alter(schema_name, &**secret)?;
 
-        secret.secret_id
+        secret.id
     };
 
     let catalog_writer = session.catalog_writer()?;
     catalog_writer.drop_secret(secret_id).await?;
 
-    Ok(RwPgResponse::builder(StatementType::DROP_SECRET)
-        .notice(format!("dropped secret \"{}\"", secret_name))
-        .into())
+    Ok(RwPgResponse::empty_result(StatementType::DROP_SECRET))
 }
diff --git a/src/frontend/src/handler/drop_sink.rs b/src/frontend/src/handler/drop_sink.rs
index 1b491be81f5ef..967fe700dfcde 100644
--- a/src/frontend/src/handler/drop_sink.rs
+++ b/src/frontend/src/handler/drop_sink.rs
@@ -12,15 +12,18 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::collections::HashSet;
+
 use pgwire::pg_response::{PgResponse, StatementType};
-use risingwave_pb::ddl_service::ReplaceTablePlan;
+use risingwave_pb::ddl_service::{ReplaceTablePlan, TableJobType};
 use risingwave_sqlparser::ast::ObjectName;
 
 use super::RwPgResponse;
 use crate::binder::Binder;
 use crate::catalog::root_catalog::SchemaPath;
 use crate::error::Result;
-use crate::handler::create_sink::{insert_merger_to_union, reparse_table_for_sink};
+use crate::handler::alter_table_column::hijack_merger_for_target_table;
+use crate::handler::create_sink::{fetch_incoming_sinks, reparse_table_for_sink};
 use crate::handler::HandlerArgs;
 
 pub async fn handle_drop_sink(
@@ -29,7 +32,7 @@ pub async fn handle_drop_sink(
     if_exists: bool,
     cascade: bool,
 ) -> Result<RwPgResponse> {
-    let session = handler_args.session;
+    let session = handler_args.session.clone();
     let db_name = session.database();
     let (schema_name, sink_name) = Binder::resolve_schema_qualified_name(db_name, sink_name)?;
     let search_path = session.config().search_path();
@@ -76,12 +79,18 @@ pub async fn handle_drop_sink(
             .incoming_sinks
             .clone_from(&table_catalog.incoming_sinks);
 
-        for _ in 0..(table_catalog.incoming_sinks.len() - 1) {
-            for fragment in graph.fragments.values_mut() {
-                if let Some(node) = &mut fragment.node {
-                    insert_merger_to_union(node);
-                }
-            }
+        let mut incoming_sink_ids: HashSet<_> =
+            table_catalog.incoming_sinks.iter().copied().collect();
+
+        assert!(incoming_sink_ids.remove(&sink_id.sink_id));
+
+        for sink in fetch_incoming_sinks(&session, &incoming_sink_ids)? {
+            hijack_merger_for_target_table(
+                &mut graph,
+                table_catalog.columns(),
+                &sink,
+                Some(&sink.unique_identity()),
+            )?;
         }
 
         affected_table_change = Some(ReplaceTablePlan {
@@ -89,6 +98,7 @@ pub async fn handle_drop_sink(
             table: Some(table),
             fragment_graph: Some(graph),
             table_col_index_mapping: None,
+            job_type: TableJobType::General as _,
         });
     }
 
diff --git a/src/frontend/src/handler/explain.rs b/src/frontend/src/handler/explain.rs
index db124b373181b..ed22462a66888 100644
--- a/src/frontend/src/handler/explain.rs
+++ b/src/frontend/src/handler/explain.rs
@@ -21,7 +21,7 @@ use thiserror_ext::AsReport;
 
 use super::create_index::{gen_create_index_plan, resolve_index_schema};
 use super::create_mv::gen_create_mv_plan;
-use super::create_sink::{gen_sink_plan, get_partition_compute_info};
+use super::create_sink::gen_sink_plan;
 use super::query::gen_batch_plan_by_statement;
 use super::util::SourceSchemaCompatExt;
 use super::{RwPgResponse, RwPgResponseBuilderExt};
@@ -87,9 +87,8 @@ async fn do_handle_explain(
                 (Ok(plan), context)
             }
             Statement::CreateSink { stmt } => {
-                let partition_info = get_partition_compute_info(&handler_args.with_options).await?;
-                let context = OptimizerContext::new(handler_args, explain_options);
-                let plan = gen_sink_plan(&session, context.into(), stmt, partition_info)
+                let plan = gen_sink_plan(handler_args, stmt, Some(explain_options))
+                    .await
                     .map(|plan| plan.sink_plan)?;
                 let context = plan.ctx();
                 (Ok(plan), context)
diff --git a/src/frontend/src/handler/extended_handle.rs b/src/frontend/src/handler/extended_handle.rs
index b497b1164d144..ac0e799d9c1ab 100644
--- a/src/frontend/src/handler/extended_handle.rs
+++ b/src/frontend/src/handler/extended_handle.rs
@@ -47,6 +47,17 @@ pub struct PreparedResult {
     pub bound_result: BoundResult,
 }
 
+/// Partial was a concept in the PostgreSQL protocol.
+///
+/// In the extended-query protocol, execution of SQL commands is divided into multiple steps.
+/// The state retained between steps is represented by two types of objects: prepared statements
+/// and portals. A prepared statement represents the result of parsing and semantic analysis of a
+/// textual query string. A prepared statement is not in itself ready to execute, because it might
+/// lack specific values for parameters.
+/// A portal represents a ready-to-execute or already-partially-executed statement,
+/// with any missing parameter values filled in.
+///
+/// Reference: <https://www.postgresql.org/docs/current/protocol-overview.html#PROTOCOL-QUERY-CONCEPTS>
 #[expect(clippy::enum_variant_names)]
 #[derive(Clone)]
 pub enum Portal {
@@ -65,6 +76,7 @@ impl std::fmt::Display for Portal {
     }
 }
 
+/// See the docs of [`Portal`].
 #[derive(Clone)]
 pub struct PortalResult {
     pub statement: Statement,
@@ -97,7 +109,14 @@ pub fn handle_parse(
         | Statement::Update { .. } => {
             query::handle_parse(handler_args, statement, specific_param_types)
         }
-        Statement::CreateView { query, .. } => {
+        Statement::CreateView {
+            query,
+            materialized,
+            ..
+        } => {
+            if *materialized {
+                return query::handle_parse(handler_args, statement, specific_param_types);
+            }
             if have_parameter_in_query(query) {
                 bail_not_implemented!("CREATE VIEW with parameters");
             }
@@ -179,13 +198,8 @@ pub async fn handle_execute(session: Arc<SessionImpl>, portal: Portal) -> Result
             let _guard = session.txn_begin_implicit(); // TODO(bugen): is this behavior correct?
             let sql: Arc<str> = Arc::from(portal.statement.to_string());
             let handler_args = HandlerArgs::new(session, &portal.statement, sql)?;
-            match &portal.statement {
-                Statement::Query(_)
-                | Statement::Insert { .. }
-                | Statement::Delete { .. }
-                | Statement::Update { .. } => query::handle_execute(handler_args, portal).await,
-                _ => unreachable!(),
-            }
+
+            query::handle_execute(handler_args, portal).await
         }
         Portal::PureStatement(stmt) => {
             let sql: Arc<str> = Arc::from(stmt.to_string());
diff --git a/src/frontend/src/handler/mod.rs b/src/frontend/src/handler/mod.rs
index f8beeedb19438..c3dcf0ce59768 100644
--- a/src/frontend/src/handler/mod.rs
+++ b/src/frontend/src/handler/mod.rs
@@ -697,7 +697,7 @@ pub async fn handle(
         } => alter_table_with_sr::handle_refresh_schema(handler_args, name).await,
         Statement::AlterTable {
             name,
-            operation: AlterTableOperation::SetStreamingRateLimit { rate_limit },
+            operation: AlterTableOperation::SetSourceRateLimit { rate_limit },
         } => {
             alter_streaming_rate_limit::handle_alter_streaming_rate_limit(
                 handler_args,
@@ -814,7 +814,7 @@ pub async fn handle(
         Statement::AlterView {
             materialized,
             name,
-            operation: AlterViewOperation::SetStreamingRateLimit { rate_limit },
+            operation: AlterViewOperation::SetBackfillRateLimit { rate_limit },
         } if materialized => {
             alter_streaming_rate_limit::handle_alter_streaming_rate_limit(
                 handler_args,
@@ -946,7 +946,7 @@ pub async fn handle(
         } => alter_source_with_sr::handler_refresh_schema(handler_args, name).await,
         Statement::AlterSource {
             name,
-            operation: AlterSourceOperation::SetStreamingRateLimit { rate_limit },
+            operation: AlterSourceOperation::SetSourceRateLimit { rate_limit },
         } => {
             alter_streaming_rate_limit::handle_alter_streaming_rate_limit(
                 handler_args,
diff --git a/src/frontend/src/handler/privilege.rs b/src/frontend/src/handler/privilege.rs
index d26d1d6d4785c..e9f60a1f78b79 100644
--- a/src/frontend/src/handler/privilege.rs
+++ b/src/frontend/src/handler/privilege.rs
@@ -115,6 +115,9 @@ pub(crate) fn resolve_privileges(stmt: &BoundStatement) -> Vec<ObjectCheckItem>
             objects.push(object);
         }
         BoundStatement::Query(ref query) => objects.extend(resolve_query_privileges(query)),
+        BoundStatement::CreateView(ref create_view) => {
+            objects.extend(resolve_query_privileges(&create_view.query))
+        }
     };
     objects
 }
diff --git a/src/frontend/src/handler/query.rs b/src/frontend/src/handler/query.rs
index 745749c9aef20..bdb32b590300b 100644
--- a/src/frontend/src/handler/query.rs
+++ b/src/frontend/src/handler/query.rs
@@ -23,14 +23,15 @@ use pgwire::pg_response::{PgResponse, StatementType};
 use pgwire::types::Format;
 use postgres_types::FromSql;
 use risingwave_batch::worker_manager::worker_node_manager::WorkerNodeSelector;
+use risingwave_common::bail_not_implemented;
 use risingwave_common::catalog::Schema;
 use risingwave_common::session_config::QueryMode;
 use risingwave_common::types::{DataType, Datum};
 use risingwave_sqlparser::ast::{SetExpr, Statement};
 
 use super::extended_handle::{PortalResult, PrepareStatement, PreparedResult};
-use super::{PgResponseStream, RwPgResponse};
-use crate::binder::{Binder, BoundStatement};
+use super::{create_mv, PgResponseStream, RwPgResponse};
+use crate::binder::{Binder, BoundCreateView, BoundStatement};
 use crate::catalog::TableId;
 use crate::error::{ErrorCode, Result, RwError};
 use crate::handler::flush::do_flush;
@@ -80,6 +81,7 @@ pub fn handle_parse(
     }))
 }
 
+/// Execute a "Portal", which is a prepared statement with bound parameters.
 pub async fn handle_execute(
     handler_args: HandlerArgs,
     portal: PortalResult,
@@ -87,17 +89,70 @@ pub async fn handle_execute(
     let PortalResult {
         bound_result,
         result_formats,
-        ..
+        statement,
     } = portal;
+    match statement {
+        Statement::Query(_)
+        | Statement::Insert { .. }
+        | Statement::Delete { .. }
+        | Statement::Update { .. } => {
+            // Execute a batch query
+            let session = handler_args.session.clone();
+            let plan_fragmenter_result = {
+                let context = OptimizerContext::from_handler_args(handler_args);
+                let plan_result = gen_batch_query_plan(&session, context.into(), bound_result)?;
+
+                gen_batch_plan_fragmenter(&session, plan_result)?
+            };
+            execute(session, plan_fragmenter_result, result_formats).await
+        }
+        Statement::CreateView { materialized, .. } if materialized => {
+            // Execute a CREATE MATERIALIZED VIEW
+            let BoundResult {
+                bound,
+                dependent_relations,
+                ..
+            } = bound_result;
+            let create_mv = if let BoundStatement::CreateView(create_mv) = bound {
+                create_mv
+            } else {
+                unreachable!("expect a BoundStatement::CreateView")
+            };
+            let BoundCreateView {
+                or_replace,
+                materialized: _,
+                if_not_exists,
+                name,
+                columns,
+                query,
+                emit_mode,
+                with_options,
+            } = *create_mv;
+            if or_replace {
+                bail_not_implemented!("CREATE OR REPLACE VIEW");
+            }
 
-    let session = handler_args.session.clone();
-    let plan_fragmenter_result = {
-        let context = OptimizerContext::from_handler_args(handler_args);
-        let plan_result = gen_batch_query_plan(&session, context.into(), bound_result)?;
+            // Hack: replace the `with_options` with the bounded ones.
+            let handler_args = HandlerArgs {
+                session: handler_args.session.clone(),
+                sql: handler_args.sql.clone(),
+                normalized_sql: handler_args.normalized_sql.clone(),
+                with_options: crate::WithOptions::try_from(with_options.as_slice())?,
+            };
 
-        gen_batch_plan_fragmenter(&session, plan_result)?
-    };
-    execute(session, plan_fragmenter_result, result_formats).await
+            create_mv::handle_create_mv_bound(
+                handler_args,
+                if_not_exists,
+                name,
+                *query,
+                dependent_relations,
+                columns,
+                emit_mode,
+            )
+            .await
+        }
+        _ => unreachable!(),
+    }
 }
 
 pub fn gen_batch_plan_by_statement(
diff --git a/src/frontend/src/handler/show.rs b/src/frontend/src/handler/show.rs
index e543617f9ddcd..5be1681e0a094 100644
--- a/src/frontend/src/handler/show.rs
+++ b/src/frontend/src/handler/show.rs
@@ -214,7 +214,7 @@ struct ShowClusterRow {
     addr: String,
     r#type: String,
     state: String,
-    parallel_units: String,
+    parallelism: i32,
     is_streaming: Option<bool>,
     is_serving: Option<bool>,
     is_unschedulable: Option<bool>,
@@ -435,7 +435,7 @@ pub async fn handle_show_object(
                     addr: addr.to_string(),
                     r#type: worker.get_type().unwrap().as_str_name().into(),
                     state: worker.get_state().unwrap().as_str_name().to_string(),
-                    parallel_units: worker.parallel_units.into_iter().map(|pu| pu.id).join(", "),
+                    parallelism: worker.get_parallelism() as _,
                     is_streaming: property.map(|p| p.is_streaming),
                     is_serving: property.map(|p| p.is_serving),
                     is_unschedulable: property.map(|p| p.is_unschedulable),
diff --git a/src/frontend/src/lib.rs b/src/frontend/src/lib.rs
index dbc312ac463ca..4bdf9fa398f77 100644
--- a/src/frontend/src/lib.rs
+++ b/src/frontend/src/lib.rs
@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#![feature(async_closure)]
 #![allow(clippy::derive_partial_eq_without_eq)]
 #![feature(map_try_insert)]
 #![feature(negative_impls)]
@@ -24,7 +25,6 @@
 #![feature(assert_matches)]
 #![feature(lint_reasons)]
 #![feature(box_patterns)]
-#![feature(lazy_cell)]
 #![feature(macro_metavar_expr)]
 #![feature(min_specialization)]
 #![feature(extend_one)]
@@ -64,7 +64,7 @@ use risingwave_common::util::meta_addr::MetaAddressStrategy;
 use risingwave_common::util::tokio_util::sync::CancellationToken;
 pub use stream_fragmenter::build_graph;
 mod utils;
-pub use utils::{explain_stream_graph, WithOptions};
+pub use utils::{explain_stream_graph, WithOptions, WithOptionsSecResolved};
 pub(crate) mod error;
 mod meta_client;
 pub mod test_utils;
@@ -142,6 +142,15 @@ pub struct FrontendOpts {
     #[clap(long, hide = true, env = "RW_ENABLE_BARRIER_READ")]
     #[override_opts(path = batch.enable_barrier_read)]
     pub enable_barrier_read: Option<bool>,
+
+    /// The path of the temp secret file directory.
+    #[clap(
+        long,
+        hide = true,
+        env = "RW_TEMP_SECRET_FILE_DIR",
+        default_value = "./secrets"
+    )]
+    pub temp_secret_file_dir: String,
 }
 
 impl risingwave_common::opts::Opts for FrontendOpts {
diff --git a/src/frontend/src/meta_client.rs b/src/frontend/src/meta_client.rs
index 56beb26207b3e..bb8a1d803a34f 100644
--- a/src/frontend/src/meta_client.rs
+++ b/src/frontend/src/meta_client.rs
@@ -33,7 +33,7 @@ use risingwave_pb::meta::list_fragment_distribution_response::FragmentDistributi
 use risingwave_pb::meta::list_object_dependencies_response::PbObjectDependencies;
 use risingwave_pb::meta::list_table_fragment_states_response::TableFragmentState;
 use risingwave_pb::meta::list_table_fragments_response::TableFragmentInfo;
-use risingwave_pb::meta::{EventLog, PbThrottleTarget};
+use risingwave_pb::meta::{EventLog, PbThrottleTarget, RecoveryStatus};
 use risingwave_rpc_client::error::Result;
 use risingwave_rpc_client::{HummockMetaClient, MetaClient};
 
@@ -126,6 +126,15 @@ pub trait FrontendMetaClient: Send + Sync {
         id: u32,
         rate_limit: Option<u32>,
     ) -> Result<()>;
+
+    async fn list_change_log_epochs(
+        &self,
+        table_id: u32,
+        min_epoch: u64,
+        max_count: u32,
+    ) -> Result<Vec<u64>>;
+
+    async fn get_cluster_recovery_status(&self) -> Result<RecoveryStatus>;
 }
 
 pub struct FrontendMetaClientImpl(pub MetaClient);
@@ -318,4 +327,19 @@ impl FrontendMetaClient for FrontendMetaClientImpl {
             .await
             .map(|_| ())
     }
+
+    async fn list_change_log_epochs(
+        &self,
+        table_id: u32,
+        min_epoch: u64,
+        max_count: u32,
+    ) -> Result<Vec<u64>> {
+        self.0
+            .list_change_log_epochs(table_id, min_epoch, max_count)
+            .await
+    }
+
+    async fn get_cluster_recovery_status(&self) -> Result<RecoveryStatus> {
+        self.0.get_cluster_recovery_status().await
+    }
 }
diff --git a/src/frontend/src/observer/observer_manager.rs b/src/frontend/src/observer/observer_manager.rs
index 38e84d213bf49..80f6316f9ca90 100644
--- a/src/frontend/src/observer/observer_manager.rs
+++ b/src/frontend/src/observer/observer_manager.rs
@@ -20,9 +20,10 @@ use parking_lot::RwLock;
 use risingwave_batch::worker_manager::worker_node_manager::WorkerNodeManagerRef;
 use risingwave_common::catalog::CatalogVersion;
 use risingwave_common::hash::WorkerSlotMapping;
+use risingwave_common::secret::LocalSecretManager;
 use risingwave_common::session_config::SessionConfig;
 use risingwave_common::system_param::local_manager::LocalSystemParamsManagerRef;
-use risingwave_common_service::observer_manager::{ObserverState, SubscribeFrontend};
+use risingwave_common_service::ObserverState;
 use risingwave_pb::common::WorkerNode;
 use risingwave_pb::hummock::HummockVersionStats;
 use risingwave_pb::meta::relation::RelationInfo;
@@ -50,7 +51,9 @@ pub struct FrontendObserverNode {
 }
 
 impl ObserverState for FrontendObserverNode {
-    type SubscribeType = SubscribeFrontend;
+    fn subscribe_type() -> risingwave_pb::meta::SubscribeType {
+        risingwave_pb::meta::SubscribeType::Frontend
+    }
 
     fn handle_notification(&mut self, resp: SubscribeResponse) {
         let Some(info) = resp.info.as_ref() else {
@@ -63,10 +66,13 @@ impl ObserverState for FrontendObserverNode {
             | Info::Schema(_)
             | Info::RelationGroup(_)
             | Info::Function(_)
-            | Info::Secret(_)
             | Info::Connection(_) => {
                 self.handle_catalog_notification(resp);
             }
+            Info::Secret(_) => {
+                self.handle_catalog_notification(resp.clone());
+                self.handle_secret_notification(resp);
+            }
             Info::Node(node) => {
                 self.update_worker_node_manager(resp.operation(), node);
             }
@@ -176,8 +182,8 @@ impl ObserverState for FrontendObserverNode {
         for connection in connections {
             catalog_guard.create_connection(&connection)
         }
-        for secret in secrets {
-            catalog_guard.create_secret(&secret)
+        for secret in &secrets {
+            catalog_guard.create_secret(secret)
         }
         for user in users {
             user_guard.create_user(user)
@@ -202,6 +208,7 @@ impl ObserverState for FrontendObserverNode {
             .unwrap();
         *self.session_params.write() =
             serde_json::from_str(&session_params.unwrap().params).unwrap();
+        LocalSecretManager::global().init_secrets(secrets);
     }
 }
 
@@ -351,16 +358,21 @@ impl FrontendObserverNode {
                 Operation::Update => catalog_guard.update_connection(connection),
                 _ => panic!("receive an unsupported notify {:?}", resp),
             },
-            Info::Secret(secret) => match resp.operation() {
-                Operation::Add => catalog_guard.create_secret(secret),
-                Operation::Delete => catalog_guard.drop_secret(
-                    secret.database_id,
-                    secret.schema_id,
-                    SecretId::new(secret.id),
-                ),
-                Operation::Update => catalog_guard.update_secret(secret),
-                _ => panic!("receive an unsupported notify {:?}", resp),
-            },
+            Info::Secret(secret) => {
+                let mut secret = secret.clone();
+                // The secret value should not be revealed to users. So mask it in the frontend catalog.
+                secret.value = "SECRET VALUE SHOULD NOT BE REVEALED".as_bytes().to_vec();
+                match resp.operation() {
+                    Operation::Add => catalog_guard.create_secret(&secret),
+                    Operation::Delete => catalog_guard.drop_secret(
+                        secret.database_id,
+                        secret.schema_id,
+                        SecretId::new(secret.id),
+                    ),
+                    Operation::Update => catalog_guard.update_secret(&secret),
+                    _ => panic!("receive an unsupported notify {:?}", resp),
+                }
+            }
             _ => unreachable!(),
         }
         assert!(
@@ -469,6 +481,24 @@ impl FrontendObserverNode {
         }
     }
 
+    fn handle_secret_notification(&mut self, resp: SubscribeResponse) {
+        let resp_op = resp.operation();
+        let Some(Info::Secret(secret)) = resp.info else {
+            unreachable!();
+        };
+        match resp_op {
+            Operation::Add => {
+                LocalSecretManager::global().add_secret(secret.id, secret.value);
+            }
+            Operation::Delete => {
+                LocalSecretManager::global().remove_secret(secret.id);
+            }
+            _ => {
+                panic!("error type notification");
+            }
+        }
+    }
+
     /// `update_worker_node_manager` is called in `start` method.
     /// It calls `add_worker_node` and `remove_worker_node` of `WorkerNodeManager`.
     fn update_worker_node_manager(&self, operation: Operation, node: WorkerNode) {
diff --git a/src/frontend/src/optimizer/logical_optimization.rs b/src/frontend/src/optimizer/logical_optimization.rs
index 931a645b3d680..de9db6f8f22d2 100644
--- a/src/frontend/src/optimizer/logical_optimization.rs
+++ b/src/frontend/src/optimizer/logical_optimization.rs
@@ -126,10 +126,22 @@ static STREAM_GENERATE_SERIES_WITH_NOW: LazyLock<OptimizationStage> = LazyLock::
     )
 });
 
-static TABLE_FUNCTION_TO_PROJECT_SET: LazyLock<OptimizationStage> = LazyLock::new(|| {
+static TABLE_FUNCTION_CONVERT: LazyLock<OptimizationStage> = LazyLock::new(|| {
     OptimizationStage::new(
-        "Table Function To Project Set",
-        vec![TableFunctionToProjectSetRule::create()],
+        "Table Function Convert",
+        vec![
+            // Apply file scan rule first
+            TableFunctionToFileScanRule::create(),
+            TableFunctionToProjectSetRule::create(),
+        ],
+        ApplyOrder::TopDown,
+    )
+});
+
+static TABLE_FUNCTION_TO_FILE_SCAN: LazyLock<OptimizationStage> = LazyLock::new(|| {
+    OptimizationStage::new(
+        "Table Function To FileScan",
+        vec![TableFunctionToFileScanRule::create()],
         ApplyOrder::TopDown,
     )
 });
@@ -584,7 +596,7 @@ impl LogicalOptimizer {
         // Should be applied before converting table function to project set.
         plan = plan.optimize_by_rules(&STREAM_GENERATE_SERIES_WITH_NOW);
         // In order to unnest a table function, we need to convert it into a `project_set` first.
-        plan = plan.optimize_by_rules(&TABLE_FUNCTION_TO_PROJECT_SET);
+        plan = plan.optimize_by_rules(&TABLE_FUNCTION_CONVERT);
 
         plan = Self::subquery_unnesting(plan, enable_share_plan, explain_trace, &ctx)?;
         if has_logical_max_one_row(plan.clone()) {
@@ -689,8 +701,10 @@ impl LogicalOptimizer {
         plan = plan.optimize_by_rules(&SET_OPERATION_MERGE);
         plan = plan.optimize_by_rules(&SET_OPERATION_TO_JOIN);
         plan = plan.optimize_by_rules(&ALWAYS_FALSE_FILTER);
+        // Table function should be converted into `file_scan` before `project_set`.
+        plan = plan.optimize_by_rules(&TABLE_FUNCTION_TO_FILE_SCAN);
         // In order to unnest a table function, we need to convert it into a `project_set` first.
-        plan = plan.optimize_by_rules(&TABLE_FUNCTION_TO_PROJECT_SET);
+        plan = plan.optimize_by_rules(&TABLE_FUNCTION_CONVERT);
 
         plan = Self::subquery_unnesting(plan, false, explain_trace, &ctx)?;
 
diff --git a/src/frontend/src/optimizer/mod.rs b/src/frontend/src/optimizer/mod.rs
index c336254910826..f59c0635b8bd2 100644
--- a/src/frontend/src/optimizer/mod.rs
+++ b/src/frontend/src/optimizer/mod.rs
@@ -81,8 +81,7 @@ use crate::optimizer::plan_node::{
 };
 use crate::optimizer::plan_visitor::TemporalJoinValidator;
 use crate::optimizer::property::Distribution;
-use crate::utils::ColIndexMappingRewriteExt;
-use crate::WithOptions;
+use crate::utils::{ColIndexMappingRewriteExt, WithOptionsSecResolved};
 
 /// `PlanRoot` is used to describe a plan. planner will construct a `PlanRoot` with `LogicalNode`.
 /// and required distribution and order. And `PlanRoot` can generate corresponding streaming or
@@ -230,7 +229,7 @@ impl PlanRoot {
         use generic::Agg;
         use plan_node::PlanAggCall;
         use risingwave_common::types::ListValue;
-        use risingwave_expr::aggregate::AggKind;
+        use risingwave_expr::aggregate::PbAggKind;
 
         use crate::expr::{ExprImpl, ExprType, FunctionCall, InputRef};
         use crate::utils::{Condition, IndexSet};
@@ -244,14 +243,13 @@ impl PlanRoot {
         let return_type = DataType::List(input_column_type.clone().into());
         let agg = Agg::new(
             vec![PlanAggCall {
-                agg_kind: AggKind::ArrayAgg,
+                agg_kind: PbAggKind::ArrayAgg.into(),
                 return_type: return_type.clone(),
                 inputs: vec![InputRef::new(select_idx, input_column_type.clone())],
                 distinct: false,
                 order_by: self.required_order.column_orders,
                 filter: Condition::true_cond(),
                 direct_args: vec![],
-                user_defined: None,
             }],
             IndexSet::empty(),
             self.plan,
@@ -926,7 +924,7 @@ impl PlanRoot {
         &mut self,
         sink_name: String,
         definition: String,
-        properties: WithOptions,
+        properties: WithOptionsSecResolved,
         emit_on_window_close: bool,
         db_name: String,
         sink_from_table_name: String,
diff --git a/src/frontend/src/optimizer/plan_expr_visitor/strong.rs b/src/frontend/src/optimizer/plan_expr_visitor/strong.rs
index 0cf1cd0a07a35..84b8c4f6eb8f3 100644
--- a/src/frontend/src/optimizer/plan_expr_visitor/strong.rs
+++ b/src/frontend/src/optimizer/plan_expr_visitor/strong.rs
@@ -188,6 +188,7 @@ impl Strong {
             | ExprType::Cot
             | ExprType::Asin
             | ExprType::Acos
+            | ExprType::Acosd
             | ExprType::Atan
             | ExprType::Atan2
             | ExprType::Sind
@@ -305,6 +306,8 @@ impl Strong {
             | ExprType::PgRelationSize
             | ExprType::PgGetSerialSequence
             | ExprType::PgIndexColumnHasProperty
+            | ExprType::PgIsInRecovery
+            | ExprType::RwRecoveryStatus
             | ExprType::IcebergTransform
             | ExprType::HasTablePrivilege
             | ExprType::HasAnyColumnPrivilege
diff --git a/src/frontend/src/optimizer/plan_node/batch_file_scan.rs b/src/frontend/src/optimizer/plan_node/batch_file_scan.rs
new file mode 100644
index 0000000000000..649c178855ef9
--- /dev/null
+++ b/src/frontend/src/optimizer/plan_node/batch_file_scan.rs
@@ -0,0 +1,103 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use pretty_xmlish::XmlNode;
+use risingwave_pb::batch_plan::file_scan_node::{FileFormat, StorageType};
+use risingwave_pb::batch_plan::plan_node::NodeBody;
+use risingwave_pb::batch_plan::FileScanNode;
+
+use super::batch::prelude::*;
+use super::utils::{childless_record, column_names_pretty, Distill};
+use super::{
+    generic, ExprRewritable, PlanBase, PlanRef, ToBatchPb, ToDistributedBatch, ToLocalBatch,
+};
+use crate::error::Result;
+use crate::optimizer::plan_node::expr_visitable::ExprVisitable;
+use crate::optimizer::property::{Distribution, Order};
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct BatchFileScan {
+    pub base: PlanBase<Batch>,
+    pub core: generic::FileScan,
+}
+
+impl BatchFileScan {
+    pub fn new(core: generic::FileScan) -> Self {
+        let base = PlanBase::new_batch_with_core(&core, Distribution::Single, Order::any());
+
+        Self { base, core }
+    }
+
+    pub fn column_names(&self) -> Vec<&str> {
+        self.schema().names_str()
+    }
+
+    pub fn clone_with_dist(&self) -> Self {
+        let base = self
+            .base
+            .clone_with_new_distribution(Distribution::SomeShard);
+        Self {
+            base,
+            core: self.core.clone(),
+        }
+    }
+}
+
+impl_plan_tree_node_for_leaf! { BatchFileScan }
+
+impl Distill for BatchFileScan {
+    fn distill<'a>(&self) -> XmlNode<'a> {
+        let fields = vec![("columns", column_names_pretty(self.schema()))];
+        childless_record("BatchFileScan", fields)
+    }
+}
+
+impl ToLocalBatch for BatchFileScan {
+    fn to_local(&self) -> Result<PlanRef> {
+        Ok(self.clone_with_dist().into())
+    }
+}
+
+impl ToDistributedBatch for BatchFileScan {
+    fn to_distributed(&self) -> Result<PlanRef> {
+        Ok(self.clone_with_dist().into())
+    }
+}
+
+impl ToBatchPb for BatchFileScan {
+    fn to_batch_prost_body(&self) -> NodeBody {
+        NodeBody::FileScan(FileScanNode {
+            columns: self
+                .core
+                .columns()
+                .into_iter()
+                .map(|col| col.to_protobuf())
+                .collect(),
+            file_format: match self.core.file_format {
+                generic::FileFormat::Parquet => FileFormat::Parquet as i32,
+            },
+            storage_type: match self.core.storage_type {
+                generic::StorageType::S3 => StorageType::S3 as i32,
+            },
+            s3_region: self.core.s3_region.clone(),
+            s3_access_key: self.core.s3_access_key.clone(),
+            s3_secret_key: self.core.s3_secret_key.clone(),
+            file_location: self.core.file_location.clone(),
+        })
+    }
+}
+
+impl ExprRewritable for BatchFileScan {}
+
+impl ExprVisitable for BatchFileScan {}
diff --git a/src/frontend/src/optimizer/plan_node/batch_iceberg_scan.rs b/src/frontend/src/optimizer/plan_node/batch_iceberg_scan.rs
index 0ac2451759543..3433feb8d210b 100644
--- a/src/frontend/src/optimizer/plan_node/batch_iceberg_scan.rs
+++ b/src/frontend/src/optimizer/plan_node/batch_iceberg_scan.rs
@@ -98,6 +98,7 @@ impl ToDistributedBatch for BatchIcebergScan {
 impl ToBatchPb for BatchIcebergScan {
     fn to_batch_prost_body(&self) -> NodeBody {
         let source_catalog = self.source_catalog().unwrap();
+        let (with_properties, secret_refs) = source_catalog.with_properties.clone().into_parts();
         NodeBody::Source(SourceNode {
             source_id: source_catalog.id,
             info: Some(source_catalog.info.clone()),
@@ -107,9 +108,9 @@ impl ToBatchPb for BatchIcebergScan {
                 .iter()
                 .map(|c| c.to_protobuf())
                 .collect(),
-            with_properties: source_catalog.with_properties.clone().into_iter().collect(),
+            with_properties,
             split: vec![],
-            secret_refs: Default::default(),
+            secret_refs,
         })
     }
 }
diff --git a/src/frontend/src/optimizer/plan_node/batch_kafka_scan.rs b/src/frontend/src/optimizer/plan_node/batch_kafka_scan.rs
index 8dca305121399..0883f3aa697cd 100644
--- a/src/frontend/src/optimizer/plan_node/batch_kafka_scan.rs
+++ b/src/frontend/src/optimizer/plan_node/batch_kafka_scan.rs
@@ -120,6 +120,7 @@ impl ToDistributedBatch for BatchKafkaScan {
 impl ToBatchPb for BatchKafkaScan {
     fn to_batch_prost_body(&self) -> NodeBody {
         let source_catalog = self.source_catalog().unwrap();
+        let (with_properties, secret_refs) = source_catalog.with_properties.clone().into_parts();
         NodeBody::Source(SourceNode {
             source_id: source_catalog.id,
             info: Some(source_catalog.info.clone()),
@@ -129,9 +130,9 @@ impl ToBatchPb for BatchKafkaScan {
                 .iter()
                 .map(|c| c.to_protobuf())
                 .collect(),
-            with_properties: source_catalog.with_properties.clone().into_iter().collect(),
+            with_properties,
             split: vec![],
-            secret_refs: Default::default(),
+            secret_refs,
         })
     }
 }
diff --git a/src/frontend/src/optimizer/plan_node/batch_lookup_join.rs b/src/frontend/src/optimizer/plan_node/batch_lookup_join.rs
index a181c590e0e65..b09f45b65574e 100644
--- a/src/frontend/src/optimizer/plan_node/batch_lookup_join.rs
+++ b/src/frontend/src/optimizer/plan_node/batch_lookup_join.rs
@@ -16,9 +16,10 @@ use pretty_xmlish::{Pretty, XmlNode};
 use risingwave_common::catalog::{ColumnId, TableDesc};
 use risingwave_pb::batch_plan::plan_node::NodeBody;
 use risingwave_pb::batch_plan::{DistributedLookupJoinNode, LocalLookupJoinNode};
+use risingwave_sqlparser::ast::AsOf;
 
 use super::batch::prelude::*;
-use super::utils::{childless_record, Distill};
+use super::utils::{childless_record, to_pb_time_travel_as_of, Distill};
 use super::{generic, ExprRewritable};
 use crate::error::Result;
 use crate::expr::{Expr, ExprRewriter, ExprVisitor};
@@ -54,6 +55,8 @@ pub struct BatchLookupJoin {
     /// If `distributed_lookup` is true, it will generate `DistributedLookupJoinNode` for
     /// `ToBatchPb`. Otherwise, it will generate `LookupJoinNode`.
     distributed_lookup: bool,
+
+    as_of: Option<AsOf>,
 }
 
 impl BatchLookupJoin {
@@ -64,6 +67,7 @@ impl BatchLookupJoin {
         right_output_column_ids: Vec<ColumnId>,
         lookup_prefix_len: usize,
         distributed_lookup: bool,
+        as_of: Option<AsOf>,
     ) -> Self {
         // We cannot create a `BatchLookupJoin` without any eq keys. We require eq keys to do the
         // lookup.
@@ -79,6 +83,7 @@ impl BatchLookupJoin {
             right_output_column_ids,
             lookup_prefix_len,
             distributed_lookup,
+            as_of,
         }
     }
 
@@ -157,6 +162,7 @@ impl PlanTreeNodeUnary for BatchLookupJoin {
             self.right_output_column_ids.clone(),
             self.lookup_prefix_len,
             self.distributed_lookup,
+            self.as_of.clone(),
         )
     }
 }
@@ -231,6 +237,7 @@ impl TryToBatchPb for BatchLookupJoin {
                 output_indices: self.core.output_indices.iter().map(|&x| x as u32).collect(),
                 null_safe: self.eq_join_predicate.null_safes(),
                 lookup_prefix_len: self.lookup_prefix_len as u32,
+                as_of: to_pb_time_travel_as_of(&self.as_of)?,
             })
         } else {
             NodeBody::LocalLookupJoin(LocalLookupJoinNode {
@@ -263,6 +270,7 @@ impl TryToBatchPb for BatchLookupJoin {
                 worker_nodes: vec![], // To be filled in at local.rs
                 null_safe: self.eq_join_predicate.null_safes(),
                 lookup_prefix_len: self.lookup_prefix_len as u32,
+                as_of: to_pb_time_travel_as_of(&self.as_of)?,
             })
         })
     }
diff --git a/src/frontend/src/optimizer/plan_node/batch_seq_scan.rs b/src/frontend/src/optimizer/plan_node/batch_seq_scan.rs
index a504df99f317f..576793f4dd450 100644
--- a/src/frontend/src/optimizer/plan_node/batch_seq_scan.rs
+++ b/src/frontend/src/optimizer/plan_node/batch_seq_scan.rs
@@ -20,9 +20,10 @@ use risingwave_common::types::ScalarImpl;
 use risingwave_common::util::scan_range::{is_full_range, ScanRange};
 use risingwave_pb::batch_plan::plan_node::NodeBody;
 use risingwave_pb::batch_plan::RowSeqScanNode;
+use risingwave_sqlparser::ast::AsOf;
 
 use super::batch::prelude::*;
-use super::utils::{childless_record, Distill};
+use super::utils::{childless_record, to_pb_time_travel_as_of, Distill};
 use super::{generic, ExprRewritable, PlanBase, PlanRef, ToDistributedBatch};
 use crate::catalog::ColumnId;
 use crate::error::Result;
@@ -39,6 +40,7 @@ pub struct BatchSeqScan {
     core: generic::TableScan,
     scan_ranges: Vec<ScanRange>,
     limit: Option<u64>,
+    as_of: Option<AsOf>,
 }
 
 impl BatchSeqScan {
@@ -68,12 +70,14 @@ impl BatchSeqScan {
                 );
             })
         }
+        let as_of = core.as_of.clone();
 
         Self {
             base,
             core,
             scan_ranges,
             limit,
+            as_of,
         }
     }
 
@@ -248,6 +252,7 @@ impl TryToBatchPb for BatchSeqScan {
             vnode_bitmap: None,
             ordered: !self.order().is_any(),
             limit: *self.limit(),
+            as_of: to_pb_time_travel_as_of(&self.as_of)?,
         }))
     }
 }
diff --git a/src/frontend/src/optimizer/plan_node/batch_source.rs b/src/frontend/src/optimizer/plan_node/batch_source.rs
index ac5e25d703636..fd33d2dba0035 100644
--- a/src/frontend/src/optimizer/plan_node/batch_source.rs
+++ b/src/frontend/src/optimizer/plan_node/batch_source.rs
@@ -102,6 +102,7 @@ impl ToDistributedBatch for BatchSource {
 impl ToBatchPb for BatchSource {
     fn to_batch_prost_body(&self) -> NodeBody {
         let source_catalog = self.source_catalog().unwrap();
+        let (with_properties, secret_refs) = source_catalog.with_properties.clone().into_parts();
         NodeBody::Source(SourceNode {
             source_id: source_catalog.id,
             info: Some(source_catalog.info.clone()),
@@ -111,9 +112,9 @@ impl ToBatchPb for BatchSource {
                 .iter()
                 .map(|c| c.to_protobuf())
                 .collect(),
-            with_properties: source_catalog.with_properties.clone().into_iter().collect(),
+            with_properties,
             split: vec![],
-            secret_refs: Default::default(),
+            secret_refs,
         })
     }
 }
diff --git a/src/frontend/src/optimizer/plan_node/generic/agg.rs b/src/frontend/src/optimizer/plan_node/generic/agg.rs
index 8f999988c4824..d1ca0177261da 100644
--- a/src/frontend/src/optimizer/plan_node/generic/agg.rs
+++ b/src/frontend/src/optimizer/plan_node/generic/agg.rs
@@ -13,7 +13,6 @@
 // limitations under the License.
 
 use std::collections::{BTreeMap, BTreeSet, HashMap};
-use std::sync::Arc;
 use std::{fmt, vec};
 
 use fixedbitset::FixedBitSet;
@@ -24,14 +23,13 @@ use risingwave_common::types::DataType;
 use risingwave_common::util::iter_util::ZipEqFast;
 use risingwave_common::util::sort_util::{ColumnOrder, ColumnOrderDisplay, OrderType};
 use risingwave_common::util::value_encoding::DatumToProtoExt;
-use risingwave_expr::aggregate::{agg_kinds, AggKind};
+use risingwave_expr::aggregate::{agg_kinds, AggKind, PbAggKind};
 use risingwave_expr::sig::{FuncBuilder, FUNCTION_REGISTRY};
 use risingwave_pb::expr::{PbAggCall, PbConstant};
 use risingwave_pb::stream_plan::{agg_call_state, AggCallState as PbAggCallState};
 
 use super::super::utils::TableCatalogBuilder;
 use super::{impl_distill_unit_from_fields, stream, GenericPlanNode, GenericPlanRef};
-use crate::catalog::function_catalog::FunctionCatalog;
 use crate::expr::{Expr, ExprRewriter, ExprVisitor, InputRef, InputRefDisplay, Literal};
 use crate::optimizer::optimizer_context::OptimizerContextRef;
 use crate::optimizer::plan_node::batch::BatchPlanRef;
@@ -107,8 +105,11 @@ impl<PlanRef: GenericPlanRef> Agg<PlanRef> {
             && !self.agg_calls.is_empty()
             && self.agg_calls.iter().all(|call| {
                 let agg_kind_ok = !matches!(call.agg_kind, agg_kinds::simply_cannot_two_phase!());
-                let order_ok = matches!(call.agg_kind, agg_kinds::result_unaffected_by_order_by!())
-                    || call.order_by.is_empty();
+                let order_ok = matches!(
+                    call.agg_kind,
+                    agg_kinds::result_unaffected_by_order_by!()
+                        | AggKind::Builtin(PbAggKind::ApproxPercentile)
+                ) || call.order_by.is_empty();
                 let distinct_ok =
                     matches!(call.agg_kind, agg_kinds::result_unaffected_by_distinct!())
                         || !call.distinct;
@@ -417,28 +418,33 @@ impl<PlanRef: stream::StreamPlanRef> Agg<PlanRef> {
                     AggCallState::Value
                 }
                 agg_kinds::single_value_state!() => AggCallState::Value,
-                AggKind::Min
-                | AggKind::Max
-                | AggKind::FirstValue
-                | AggKind::LastValue
-                | AggKind::StringAgg
-                | AggKind::ArrayAgg
-                | AggKind::JsonbAgg
-                | AggKind::JsonbObjectAgg => {
+                AggKind::Builtin(
+                    PbAggKind::Min
+                    | PbAggKind::Max
+                    | PbAggKind::FirstValue
+                    | PbAggKind::LastValue
+                    | PbAggKind::StringAgg
+                    | PbAggKind::ArrayAgg
+                    | PbAggKind::JsonbAgg
+                    | PbAggKind::JsonbObjectAgg,
+                )
+                | AggKind::WrapScalar(_) => {
                     // columns with order requirement in state table
                     let sort_keys = {
                         match agg_call.agg_kind {
-                            AggKind::Min => {
+                            AggKind::Builtin(PbAggKind::Min) => {
                                 vec![(OrderType::ascending(), agg_call.inputs[0].index)]
                             }
-                            AggKind::Max => {
+                            AggKind::Builtin(PbAggKind::Max) => {
                                 vec![(OrderType::descending(), agg_call.inputs[0].index)]
                             }
-                            AggKind::FirstValue
-                            | AggKind::LastValue
-                            | AggKind::StringAgg
-                            | AggKind::ArrayAgg
-                            | AggKind::JsonbAgg => {
+                            AggKind::Builtin(
+                                PbAggKind::FirstValue
+                                | PbAggKind::LastValue
+                                | PbAggKind::StringAgg
+                                | PbAggKind::ArrayAgg
+                                | PbAggKind::JsonbAgg,
+                            ) => {
                                 if agg_call.order_by.is_empty() {
                                     me.ctx().warn_to_user(format!(
                                         "{} without ORDER BY may produce non-deterministic result",
@@ -450,7 +456,10 @@ impl<PlanRef: stream::StreamPlanRef> Agg<PlanRef> {
                                     .iter()
                                     .map(|o| {
                                         (
-                                            if agg_call.agg_kind == AggKind::LastValue {
+                                            if matches!(
+                                                agg_call.agg_kind,
+                                                AggKind::Builtin(PbAggKind::LastValue)
+                                            ) {
                                                 o.order_type.reverse()
                                             } else {
                                                 o.order_type
@@ -460,7 +469,8 @@ impl<PlanRef: stream::StreamPlanRef> Agg<PlanRef> {
                                     })
                                     .collect()
                             }
-                            AggKind::JsonbObjectAgg => agg_call
+                            AggKind::Builtin(PbAggKind::JsonbObjectAgg)
+                            | AggKind::WrapScalar(_) => agg_call
                                 .order_by
                                 .iter()
                                 .map(|o| (o.order_type, o.column_index))
@@ -481,14 +491,14 @@ impl<PlanRef: stream::StreamPlanRef> Agg<PlanRef> {
 
                     // other columns that should be contained in state table
                     let include_keys = match agg_call.agg_kind {
-                        AggKind::FirstValue
-                        | AggKind::LastValue
-                        | AggKind::StringAgg
-                        | AggKind::ArrayAgg
-                        | AggKind::JsonbAgg
-                        | AggKind::JsonbObjectAgg => {
-                            agg_call.inputs.iter().map(|i| i.index).collect()
-                        }
+                        AggKind::Builtin(
+                            PbAggKind::FirstValue
+                            | PbAggKind::LastValue
+                            | PbAggKind::StringAgg
+                            | PbAggKind::ArrayAgg
+                            | PbAggKind::JsonbAgg
+                            | PbAggKind::JsonbObjectAgg,
+                        ) => agg_call.inputs.iter().map(|i| i.index).collect(),
                         _ => vec![],
                     };
 
@@ -501,6 +511,11 @@ impl<PlanRef: stream::StreamPlanRef> Agg<PlanRef> {
                 agg_kinds::unimplemented_in_stream!() => {
                     unreachable!("should have been banned")
                 }
+                AggKind::Builtin(
+                    PbAggKind::Unspecified | PbAggKind::UserDefined | PbAggKind::WrapScalar,
+                ) => {
+                    unreachable!("invalid agg kind")
+                }
             })
             .collect()
     }
@@ -522,14 +537,21 @@ impl<PlanRef: stream::StreamPlanRef> Agg<PlanRef> {
             .iter()
             .zip_eq_fast(&mut out_fields[self.group_key.len()..])
         {
-            if agg_call.agg_kind == AggKind::UserDefined {
-                // for user defined aggregate, the state type is always BYTEA
-                field.data_type = DataType::Bytea;
-                continue;
-            }
+            let agg_kind = match agg_call.agg_kind {
+                AggKind::UserDefined(_) => {
+                    // for user defined aggregate, the state type is always BYTEA
+                    field.data_type = DataType::Bytea;
+                    continue;
+                }
+                AggKind::WrapScalar(_) => {
+                    // for wrapped scalar function, the state is always NULL
+                    continue;
+                }
+                AggKind::Builtin(kind) => kind,
+            };
             let sig = FUNCTION_REGISTRY
                 .get(
-                    agg_call.agg_kind,
+                    agg_kind,
                     &agg_call
                         .inputs
                         .iter()
@@ -710,9 +732,6 @@ pub struct PlanAggCall {
     /// `filter` evaluates to `true` will be fed to the aggregate function.
     pub filter: Condition,
     pub direct_args: Vec<Literal>,
-
-    /// Catalog of user defined aggregate function.
-    pub user_defined: Option<Arc<FunctionCatalog>>,
 }
 
 impl fmt::Debug for PlanAggCall {
@@ -767,7 +786,12 @@ impl PlanAggCall {
 
     pub fn to_protobuf(&self) -> PbAggCall {
         PbAggCall {
-            r#type: self.agg_kind.to_protobuf().into(),
+            r#type: match &self.agg_kind {
+                AggKind::Builtin(kind) => *kind,
+                AggKind::UserDefined(_) => PbAggKind::UserDefined,
+                AggKind::WrapScalar(_) => PbAggKind::WrapScalar,
+            }
+            .into(),
             return_type: Some(self.return_type.to_protobuf()),
             args: self.inputs.iter().map(InputRef::to_proto).collect(),
             distinct: self.distinct,
@@ -781,7 +805,14 @@ impl PlanAggCall {
                     r#type: Some(x.return_type().to_protobuf()),
                 })
                 .collect(),
-            udf: self.user_defined.as_ref().map(|c| c.as_ref().into()),
+            udf: match &self.agg_kind {
+                AggKind::UserDefined(udf) => Some(udf.clone()),
+                _ => None,
+            },
+            scalar: match &self.agg_kind {
+                AggKind::WrapScalar(expr) => Some(expr.clone()),
+                _ => None,
+            },
         }
     }
 
@@ -801,14 +832,13 @@ impl PlanAggCall {
 
     pub fn count_star() -> Self {
         PlanAggCall {
-            agg_kind: AggKind::Count,
+            agg_kind: PbAggKind::Count.into(),
             return_type: DataType::Int64,
             inputs: vec![],
             distinct: false,
             order_by: vec![],
             filter: Condition::true_cond(),
             direct_args: vec![],
-            user_defined: None,
         }
     }
 
diff --git a/src/frontend/src/optimizer/plan_node/generic/cdc_scan.rs b/src/frontend/src/optimizer/plan_node/generic/cdc_scan.rs
index 2d7d708291e47..ff1018de2e633 100644
--- a/src/frontend/src/optimizer/plan_node/generic/cdc_scan.rs
+++ b/src/frontend/src/optimizer/plan_node/generic/cdc_scan.rs
@@ -33,7 +33,7 @@ use crate::catalog::ColumnId;
 use crate::error::Result;
 use crate::expr::{ExprRewriter, ExprVisitor};
 use crate::optimizer::optimizer_context::OptimizerContextRef;
-use crate::optimizer::property::FunctionalDependencySet;
+use crate::optimizer::property::{FunctionalDependencySet, MonotonicityMap};
 use crate::WithOptions;
 
 /// [`CdcScan`] reads rows of a table from an external upstream database
@@ -125,6 +125,10 @@ impl CdcScan {
         FixedBitSet::with_capacity(self.get_table_columns().len())
     }
 
+    pub fn columns_monotonicity(&self) -> MonotonicityMap {
+        MonotonicityMap::new()
+    }
+
     pub(crate) fn column_names_with_table_prefix(&self) -> Vec<String> {
         self.output_col_idx
             .iter()
diff --git a/src/frontend/src/optimizer/plan_node/generic/dynamic_filter.rs b/src/frontend/src/optimizer/plan_node/generic/dynamic_filter.rs
index 1f6ab5be98b9e..4112bd0a60d8c 100644
--- a/src/frontend/src/optimizer/plan_node/generic/dynamic_filter.rs
+++ b/src/frontend/src/optimizer/plan_node/generic/dynamic_filter.rs
@@ -12,7 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use fixedbitset::FixedBitSet;
 use pretty_xmlish::Pretty;
 use risingwave_common::catalog::Schema;
 use risingwave_common::util::sort_util::OrderType;
@@ -31,9 +30,9 @@ pub struct DynamicFilter<PlanRef> {
     /// The predicate (formed with exactly one of < , <=, >, >=)
     comparator: ExprType,
     left_index: usize,
-    pub left: PlanRef,
+    left: PlanRef,
     /// The right input can only have one column.
-    pub right: PlanRef,
+    right: PlanRef,
 }
 impl<PlanRef> DynamicFilter<PlanRef> {
     pub fn comparator(&self) -> ExprType {
@@ -108,19 +107,6 @@ impl<PlanRef: GenericPlanRef> DynamicFilter<PlanRef> {
         }
     }
 
-    pub fn watermark_columns(&self, right_watermark: bool) -> FixedBitSet {
-        let mut watermark_columns = FixedBitSet::with_capacity(self.left.schema().len());
-        if right_watermark {
-            match self.comparator {
-                ExprType::Equal | ExprType::GreaterThan | ExprType::GreaterThanOrEqual => {
-                    watermark_columns.set(self.left_index, true)
-                }
-                _ => {}
-            }
-        }
-        watermark_columns
-    }
-
     fn condition_display(&self) -> (Condition, Schema) {
         let mut concat_schema = self.left.schema().fields.clone();
         concat_schema.extend(self.right.schema().fields.clone());
diff --git a/src/frontend/src/optimizer/plan_node/generic/file_scan.rs b/src/frontend/src/optimizer/plan_node/generic/file_scan.rs
new file mode 100644
index 0000000000000..975151d89c797
--- /dev/null
+++ b/src/frontend/src/optimizer/plan_node/generic/file_scan.rs
@@ -0,0 +1,77 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use educe::Educe;
+use risingwave_common::catalog::{ColumnDesc, ColumnId, Schema};
+
+use super::GenericPlanNode;
+use crate::optimizer::optimizer_context::OptimizerContextRef;
+use crate::optimizer::property::FunctionalDependencySet;
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub enum FileFormat {
+    Parquet,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub enum StorageType {
+    S3,
+}
+
+#[derive(Debug, Clone, Educe)]
+#[educe(PartialEq, Eq, Hash)]
+pub struct FileScan {
+    pub schema: Schema,
+    pub file_format: FileFormat,
+    pub storage_type: StorageType,
+    pub s3_region: String,
+    pub s3_access_key: String,
+    pub s3_secret_key: String,
+    pub file_location: Vec<String>,
+
+    #[educe(PartialEq(ignore))]
+    #[educe(Hash(ignore))]
+    pub ctx: OptimizerContextRef,
+}
+
+impl GenericPlanNode for FileScan {
+    fn schema(&self) -> Schema {
+        self.schema.clone()
+    }
+
+    fn stream_key(&self) -> Option<Vec<usize>> {
+        None
+    }
+
+    fn ctx(&self) -> OptimizerContextRef {
+        self.ctx.clone()
+    }
+
+    fn functional_dependency(&self) -> FunctionalDependencySet {
+        FunctionalDependencySet::new(self.schema.len())
+    }
+}
+
+impl FileScan {
+    pub fn columns(&self) -> Vec<ColumnDesc> {
+        self.schema
+            .fields
+            .iter()
+            .enumerate()
+            .map(|(i, f)| {
+                ColumnDesc::named(f.name.clone(), ColumnId::new(i as i32), f.data_type.clone())
+            })
+            .collect()
+    }
+}
diff --git a/src/frontend/src/optimizer/plan_node/generic/mod.rs b/src/frontend/src/optimizer/plan_node/generic/mod.rs
index 38efb6fe2a27d..3e01dee8aa0b9 100644
--- a/src/frontend/src/optimizer/plan_node/generic/mod.rs
+++ b/src/frontend/src/optimizer/plan_node/generic/mod.rs
@@ -12,6 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//! This module contains the generic plan nodes that are shared by all the plan nodes.
+//! They are meant to reuse the common fields between logical, batch and stream nodes.
+
 use std::borrow::Cow;
 use std::hash::Hash;
 
@@ -83,6 +86,9 @@ pub use changelog::*;
 mod now;
 pub use now::*;
 
+mod file_scan;
+pub use file_scan::*;
+
 pub trait DistillUnit {
     fn distill_with_name<'a>(&self, name: impl Into<Cow<'a, str>>) -> XmlNode<'a>;
 }
diff --git a/src/frontend/src/optimizer/plan_node/generic/over_window.rs b/src/frontend/src/optimizer/plan_node/generic/over_window.rs
index 2b9a748e180d3..c39fd99be9895 100644
--- a/src/frontend/src/optimizer/plan_node/generic/over_window.rs
+++ b/src/frontend/src/optimizer/plan_node/generic/over_window.rs
@@ -115,7 +115,7 @@ impl PlanWindowFunction {
         use risingwave_pb::expr::window_function::{PbGeneralType, PbType};
         use WindowFuncKind::*;
 
-        let r#type = match self.kind {
+        let r#type = match &self.kind {
             RowNumber => PbType::General(PbGeneralType::RowNumber as _),
             Rank => PbType::General(PbGeneralType::Rank as _),
             DenseRank => PbType::General(PbGeneralType::DenseRank as _),
diff --git a/src/frontend/src/optimizer/plan_node/logical_agg.rs b/src/frontend/src/optimizer/plan_node/logical_agg.rs
index f32b37d250b5a..987a0ae204869 100644
--- a/src/frontend/src/optimizer/plan_node/logical_agg.rs
+++ b/src/frontend/src/optimizer/plan_node/logical_agg.rs
@@ -15,16 +15,16 @@
 use fixedbitset::FixedBitSet;
 use itertools::Itertools;
 use risingwave_common::types::{DataType, Datum, ScalarImpl};
-use risingwave_common::util::sort_util::ColumnOrder;
-use risingwave_common::{bail_not_implemented, not_implemented};
-use risingwave_expr::aggregate::{agg_kinds, AggKind};
+use risingwave_common::util::sort_util::{ColumnOrder, OrderType};
+use risingwave_common::{bail, bail_not_implemented, not_implemented};
+use risingwave_expr::aggregate::{agg_kinds, AggKind, PbAggKind};
 
 use super::generic::{self, Agg, GenericPlanRef, PlanAggCall, ProjectBuilder};
 use super::utils::impl_distill_by_unit;
 use super::{
     BatchHashAgg, BatchSimpleAgg, ColPrunable, ExprRewritable, Logical, PlanBase, PlanRef,
-    PlanTreeNodeUnary, PredicatePushdown, StreamHashAgg, StreamProject, StreamSimpleAgg,
-    StreamStatelessSimpleAgg, ToBatch, ToStream,
+    PlanTreeNodeUnary, PredicatePushdown, StreamHashAgg, StreamProject, StreamShare,
+    StreamSimpleAgg, StreamStatelessSimpleAgg, ToBatch, ToStream,
 };
 use crate::error::{ErrorCode, Result, RwError};
 use crate::expr::{
@@ -33,6 +33,9 @@ use crate::expr::{
 };
 use crate::optimizer::plan_node::expr_visitable::ExprVisitable;
 use crate::optimizer::plan_node::generic::GenericPlanNode;
+use crate::optimizer::plan_node::stream_global_approx_percentile::StreamGlobalApproxPercentile;
+use crate::optimizer::plan_node::stream_local_approx_percentile::StreamLocalApproxPercentile;
+use crate::optimizer::plan_node::stream_row_merge::StreamRowMerge;
 use crate::optimizer::plan_node::{
     gen_filter_and_pushdown, BatchSortAgg, ColumnPruningContext, LogicalDedup, LogicalProject,
     PredicatePushdownContext, RewriteStreamContext, ToStreamContext,
@@ -42,6 +45,17 @@ use crate::utils::{
     ColIndexMapping, ColIndexMappingRewriteExt, Condition, GroupBy, IndexSet, Substitute,
 };
 
+pub struct AggInfo {
+    pub calls: Vec<PlanAggCall>,
+    pub col_mapping: ColIndexMapping,
+}
+
+/// `SeparatedAggInfo` is used to separate normal and approx percentile aggs.
+pub struct SeparatedAggInfo {
+    normal: AggInfo,
+    approx: AggInfo,
+}
+
 /// `LogicalAgg` groups input data by their group key and computes aggregation functions.
 ///
 /// It corresponds to the `GROUP BY` operator in a SQL query statement together with the aggregate
@@ -60,22 +74,70 @@ impl LogicalAgg {
     fn gen_stateless_two_phase_streaming_agg_plan(&self, stream_input: PlanRef) -> Result<PlanRef> {
         debug_assert!(self.group_key().is_empty());
         let mut core = self.core.clone();
-        core.input = stream_input;
+
+        // ====== Handle approx percentile aggs
+        let SeparatedAggInfo { normal, approx } = self.separate_normal_and_special_agg();
+
+        let AggInfo {
+            calls: non_approx_percentile_agg_calls,
+            col_mapping: non_approx_percentile_col_mapping,
+        } = normal;
+        let AggInfo {
+            calls: approx_percentile_agg_calls,
+            col_mapping: approx_percentile_col_mapping,
+        } = approx;
+
+        let needs_row_merge = (!non_approx_percentile_agg_calls.is_empty()
+            && !approx_percentile_agg_calls.is_empty())
+            || approx_percentile_agg_calls.len() >= 2;
+        core.input = if needs_row_merge {
+            // If there's row merge, we need to share the input.
+            StreamShare::new_from_input(stream_input.clone()).into()
+        } else {
+            stream_input
+        };
+        core.agg_calls = non_approx_percentile_agg_calls;
+
+        let approx_percentile =
+            self.build_approx_percentile_aggs(core.input.clone(), &approx_percentile_agg_calls)?;
+
+        // ====== Handle normal aggs
+        if core.agg_calls.is_empty() {
+            if let Some(approx_percentile) = approx_percentile {
+                return Ok(approx_percentile);
+            };
+            bail!("expected at least one agg call");
+        }
+        let total_agg_calls = core
+            .agg_calls
+            .iter()
+            .enumerate()
+            .map(|(partial_output_idx, agg_call)| {
+                agg_call.partial_to_total_agg_call(partial_output_idx)
+            })
+            .collect_vec();
         let local_agg = StreamStatelessSimpleAgg::new(core);
         let exchange =
             RequiredDist::single().enforce_if_not_satisfies(local_agg.into(), &Order::any())?;
-        let global_agg = new_stream_simple_agg(Agg::new(
-            self.agg_calls()
-                .iter()
-                .enumerate()
-                .map(|(partial_output_idx, agg_call)| {
-                    agg_call.partial_to_total_agg_call(partial_output_idx)
-                })
-                .collect(),
-            IndexSet::empty(),
-            exchange,
-        ));
-        Ok(global_agg.into())
+        let global_agg =
+            new_stream_simple_agg(Agg::new(total_agg_calls, IndexSet::empty(), exchange));
+
+        // ====== Merge approx percentile and normal aggs
+        if let Some(approx_percentile) = approx_percentile {
+            if needs_row_merge {
+                let row_merge = StreamRowMerge::new(
+                    approx_percentile,
+                    global_agg.into(),
+                    approx_percentile_col_mapping,
+                    non_approx_percentile_col_mapping,
+                )?;
+                Ok(row_merge.into())
+            } else {
+                Ok(approx_percentile)
+            }
+        } else {
+            Ok(global_agg.into())
+        }
     }
 
     /// Generate plan for stateless/stateful 2-phase streaming agg.
@@ -242,6 +304,94 @@ impl LogicalAgg {
         }
     }
 
+    fn separate_normal_and_special_agg(&self) -> SeparatedAggInfo {
+        let estimated_len = self.agg_calls().len() - 1;
+        let mut approx_percentile_agg_calls = Vec::with_capacity(estimated_len);
+        let mut non_approx_percentile_agg_calls = Vec::with_capacity(estimated_len);
+        let mut approx_percentile_col_mapping = Vec::with_capacity(estimated_len);
+        let mut non_approx_percentile_col_mapping = Vec::with_capacity(estimated_len);
+        for (output_idx, agg_call) in self.agg_calls().iter().enumerate() {
+            if agg_call.agg_kind == AggKind::Builtin(PbAggKind::ApproxPercentile) {
+                approx_percentile_agg_calls.push(agg_call.clone());
+                approx_percentile_col_mapping.push(Some(output_idx));
+            } else {
+                non_approx_percentile_agg_calls.push(agg_call.clone());
+                non_approx_percentile_col_mapping.push(Some(output_idx));
+            }
+        }
+        let normal = AggInfo {
+            calls: non_approx_percentile_agg_calls,
+            col_mapping: ColIndexMapping::new(
+                non_approx_percentile_col_mapping,
+                self.agg_calls().len(),
+            ),
+        };
+        let approx = AggInfo {
+            calls: approx_percentile_agg_calls,
+            col_mapping: ColIndexMapping::new(
+                approx_percentile_col_mapping,
+                self.agg_calls().len(),
+            ),
+        };
+        SeparatedAggInfo { normal, approx }
+    }
+
+    fn build_approx_percentile_agg(
+        &self,
+        input: PlanRef,
+        approx_percentile_agg_call: &PlanAggCall,
+    ) -> Result<PlanRef> {
+        let local_approx_percentile =
+            StreamLocalApproxPercentile::new(input, approx_percentile_agg_call);
+        let exchange = RequiredDist::single()
+            .enforce_if_not_satisfies(local_approx_percentile.into(), &Order::any())?;
+        let global_approx_percentile =
+            StreamGlobalApproxPercentile::new(exchange, approx_percentile_agg_call);
+        Ok(global_approx_percentile.into())
+    }
+
+    /// If only 1 approx percentile, just return it.
+    /// Otherwise build a tree of approx percentile with `MergeProject`.
+    /// e.g.
+    /// ApproxPercentile(col1, 0.5) as x,
+    /// ApproxPercentile(col2, 0.5) as y,
+    /// ApproxPercentile(col3, 0.5) as z
+    /// will be built as
+    ///        `MergeProject`
+    ///       /          \
+    ///  `MergeProject`       z
+    ///  /        \
+    /// x          y
+
+    fn build_approx_percentile_aggs(
+        &self,
+        input: PlanRef,
+        approx_percentile_agg_call: &[PlanAggCall],
+    ) -> Result<Option<PlanRef>> {
+        if approx_percentile_agg_call.is_empty() {
+            return Ok(None);
+        }
+        let approx_percentile_plans: Vec<PlanRef> = approx_percentile_agg_call
+            .iter()
+            .map(|agg_call| self.build_approx_percentile_agg(input.clone(), agg_call))
+            .try_collect()?;
+        assert!(!approx_percentile_plans.is_empty());
+        let mut iter = approx_percentile_plans.into_iter();
+        let mut acc = iter.next().unwrap();
+        for (current_size, plan) in iter.enumerate().map(|(i, p)| (i + 1, p)) {
+            let new_size = current_size + 1;
+            let row_merge = StreamRowMerge::new(
+                acc,
+                plan,
+                ColIndexMapping::identity_or_none(current_size, new_size),
+                ColIndexMapping::new(vec![Some(current_size)], new_size),
+            )
+            .expect("failed to build row merge");
+            acc = row_merge.into();
+        }
+        Ok(Some(acc))
+    }
+
     pub fn core(&self) -> &Agg<PlanRef> {
         &self.core
     }
@@ -384,11 +534,11 @@ impl LogicalAggBuilder {
     ) -> Result<ExprImpl> {
         match agg_call.agg_kind {
             // Rewrite avg to cast(sum as avg_return_type) / count.
-            AggKind::Avg => {
+            AggKind::Builtin(PbAggKind::Avg) => {
                 assert_eq!(agg_call.args.len(), 1);
 
                 let sum = ExprImpl::from(push_agg_call(AggCall::new(
-                    AggKind::Sum,
+                    PbAggKind::Sum.into(),
                     agg_call.args.clone(),
                     agg_call.distinct,
                     agg_call.order_by.clone(),
@@ -398,7 +548,7 @@ impl LogicalAggBuilder {
                 .cast_explicit(agg_call.return_type())?;
 
                 let count = ExprImpl::from(push_agg_call(AggCall::new(
-                    AggKind::Count,
+                    PbAggKind::Count.into(),
                     agg_call.args.clone(),
                     agg_call.distinct,
                     agg_call.order_by.clone(),
@@ -416,10 +566,12 @@ impl LogicalAggBuilder {
             // which is in a sense more general than the pow function, especially when calculating
             // covariances in the future. Also we don't have the sqrt function for rooting, so we
             // use pow(x, 0.5) to simulate
-            kind @ (AggKind::StddevPop
-            | AggKind::StddevSamp
-            | AggKind::VarPop
-            | AggKind::VarSamp) => {
+            AggKind::Builtin(
+                kind @ (PbAggKind::StddevPop
+                | PbAggKind::StddevSamp
+                | PbAggKind::VarPop
+                | PbAggKind::VarSamp),
+            ) => {
                 let arg = agg_call.args().iter().exactly_one().unwrap();
                 let squared_arg = ExprImpl::from(FunctionCall::new(
                     ExprType::Multiply,
@@ -427,7 +579,7 @@ impl LogicalAggBuilder {
                 )?);
 
                 let sum_of_sq = ExprImpl::from(push_agg_call(AggCall::new(
-                    AggKind::Sum,
+                    PbAggKind::Sum.into(),
                     vec![squared_arg],
                     agg_call.distinct,
                     agg_call.order_by.clone(),
@@ -437,7 +589,7 @@ impl LogicalAggBuilder {
                 .cast_explicit(agg_call.return_type())?;
 
                 let sum = ExprImpl::from(push_agg_call(AggCall::new(
-                    AggKind::Sum,
+                    PbAggKind::Sum.into(),
                     agg_call.args.clone(),
                     agg_call.distinct,
                     agg_call.order_by.clone(),
@@ -447,7 +599,7 @@ impl LogicalAggBuilder {
                 .cast_explicit(agg_call.return_type())?;
 
                 let count = ExprImpl::from(push_agg_call(AggCall::new(
-                    AggKind::Count,
+                    PbAggKind::Count.into(),
                     agg_call.args.clone(),
                     agg_call.distinct,
                     agg_call.order_by.clone(),
@@ -477,11 +629,10 @@ impl LogicalAggBuilder {
                 )?);
 
                 let denominator = match kind {
-                    AggKind::VarPop | AggKind::StddevPop => count.clone(),
-                    AggKind::VarSamp | AggKind::StddevSamp => ExprImpl::from(FunctionCall::new(
-                        ExprType::Subtract,
-                        vec![count.clone(), one.clone()],
-                    )?),
+                    PbAggKind::VarPop | PbAggKind::StddevPop => count.clone(),
+                    PbAggKind::VarSamp | PbAggKind::StddevSamp => ExprImpl::from(
+                        FunctionCall::new(ExprType::Subtract, vec![count.clone(), one.clone()])?,
+                    ),
                     _ => unreachable!(),
                 };
 
@@ -490,13 +641,13 @@ impl LogicalAggBuilder {
                     vec![numerator, denominator],
                 )?);
 
-                if matches!(kind, AggKind::StddevPop | AggKind::StddevSamp) {
+                if matches!(kind, PbAggKind::StddevPop | PbAggKind::StddevSamp) {
                     target = ExprImpl::from(FunctionCall::new(ExprType::Sqrt, vec![target])?);
                 }
 
                 match kind {
-                    AggKind::VarPop | AggKind::StddevPop => Ok(target),
-                    AggKind::StddevSamp | AggKind::VarSamp => {
+                    PbAggKind::VarPop | PbAggKind::StddevPop => Ok(target),
+                    PbAggKind::StddevSamp | PbAggKind::VarSamp => {
                         let case_cond = ExprImpl::from(FunctionCall::new(
                             ExprType::LessThanOrEqual,
                             vec![count, one],
@@ -511,6 +662,35 @@ impl LogicalAggBuilder {
                     _ => unreachable!(),
                 }
             }
+            AggKind::Builtin(PbAggKind::ApproxPercentile) => {
+                if agg_call.order_by.sort_exprs[0].order_type == OrderType::descending() {
+                    // Rewrite DESC into 1.0-percentile for approx_percentile.
+                    let prev_percentile = agg_call.direct_args[0].clone();
+                    let new_percentile = 1.0
+                        - prev_percentile
+                            .get_data()
+                            .as_ref()
+                            .unwrap()
+                            .as_float64()
+                            .into_inner();
+                    let new_percentile = Some(ScalarImpl::Float64(new_percentile.into()));
+                    let new_percentile = Literal::new(new_percentile, DataType::Float64);
+                    let new_direct_args = vec![new_percentile, agg_call.direct_args[1].clone()];
+
+                    let new_agg_call = AggCall {
+                        order_by: OrderBy::any(),
+                        direct_args: new_direct_args,
+                        ..agg_call
+                    };
+                    Ok(push_agg_call(new_agg_call)?.into())
+                } else {
+                    let new_agg_call = AggCall {
+                        order_by: OrderBy::any(),
+                        ..agg_call
+                    };
+                    Ok(push_agg_call(new_agg_call)?.into())
+                }
+            }
             _ => Ok(push_agg_call(agg_call)?.into()),
         }
     }
@@ -527,7 +707,6 @@ impl LogicalAggBuilder {
             order_by,
             filter,
             direct_args,
-            user_defined,
         } = agg_call;
 
         self.is_in_filter_clause = true;
@@ -565,7 +744,6 @@ impl LogicalAggBuilder {
             order_by,
             filter,
             direct_args,
-            user_defined,
         };
 
         if let Some((pos, existing)) = self
@@ -615,7 +793,7 @@ impl LogicalAggBuilder {
             agg_call.distinct = false;
         }
 
-        if matches!(agg_call.agg_kind, AggKind::Grouping) {
+        if matches!(agg_call.agg_kind, AggKind::Builtin(PbAggKind::Grouping)) {
             if self.grouping_sets.is_empty() {
                 return Err(ErrorCode::NotSupported(
                     "GROUPING must be used in a query with grouping sets".into(),
@@ -1131,8 +1309,26 @@ impl ToStream for LogicalAgg {
                 },
                 final_agg.agg_calls().len(),
             )
+        } else if let Some(_approx_percentile_agg) = plan.as_stream_global_approx_percentile() {
+            if eowc {
+                return Err(ErrorCode::InvalidInputSyntax(
+                    "`EMIT ON WINDOW CLOSE` cannot be used for aggregation without `GROUP BY`"
+                        .to_string(),
+                )
+                .into());
+            }
+            (plan.clone(), 1)
+        } else if let Some(stream_row_merge) = plan.as_stream_row_merge() {
+            if eowc {
+                return Err(ErrorCode::InvalidInputSyntax(
+                    "`EMIT ON WINDOW CLOSE` cannot be used for aggregation without `GROUP BY`"
+                        .to_string(),
+                )
+                .into());
+            }
+            (plan.clone(), stream_row_merge.base.schema().len())
         } else {
-            panic!("the root PlanNode must be either StreamHashAgg or StreamSimpleAgg");
+            panic!("the root PlanNode must be StreamHashAgg, StreamSimpleAgg, StreamGlobalApproxPercentile, or StreamRowMerge");
         };
 
         if self.agg_calls().len() == n_final_agg_calls {
@@ -1225,7 +1421,7 @@ mod tests {
         // Test case: select v1, min(v2) from test group by v1;
         {
             let min_v2 = AggCall::new(
-                AggKind::Min,
+                PbAggKind::Min.into(),
                 vec![input_ref_2.clone().into()],
                 false,
                 OrderBy::any(),
@@ -1243,7 +1439,7 @@ mod tests {
             assert_eq_input_ref!(&exprs[1], 1);
 
             assert_eq!(agg_calls.len(), 1);
-            assert_eq!(agg_calls[0].agg_kind, AggKind::Min);
+            assert_eq!(agg_calls[0].agg_kind, PbAggKind::Min.into());
             assert_eq!(input_ref_to_column_indices(&agg_calls[0].inputs), vec![1]);
             assert_eq!(group_key, vec![0].into());
         }
@@ -1251,7 +1447,7 @@ mod tests {
         // Test case: select v1, min(v2) + max(v3) from t group by v1;
         {
             let min_v2 = AggCall::new(
-                AggKind::Min,
+                PbAggKind::Min.into(),
                 vec![input_ref_2.clone().into()],
                 false,
                 OrderBy::any(),
@@ -1260,7 +1456,7 @@ mod tests {
             )
             .unwrap();
             let max_v3 = AggCall::new(
-                AggKind::Max,
+                PbAggKind::Max.into(),
                 vec![input_ref_3.clone().into()],
                 false,
                 OrderBy::any(),
@@ -1286,9 +1482,9 @@ mod tests {
             }
 
             assert_eq!(agg_calls.len(), 2);
-            assert_eq!(agg_calls[0].agg_kind, AggKind::Min);
+            assert_eq!(agg_calls[0].agg_kind, PbAggKind::Min.into());
             assert_eq!(input_ref_to_column_indices(&agg_calls[0].inputs), vec![1]);
-            assert_eq!(agg_calls[1].agg_kind, AggKind::Max);
+            assert_eq!(agg_calls[1].agg_kind, PbAggKind::Max.into());
             assert_eq!(input_ref_to_column_indices(&agg_calls[1].inputs), vec![2]);
             assert_eq!(group_key, vec![0].into());
         }
@@ -1301,7 +1497,7 @@ mod tests {
             )
             .unwrap();
             let agg_call = AggCall::new(
-                AggKind::Min,
+                PbAggKind::Min.into(),
                 vec![v1_mult_v3.into()],
                 false,
                 OrderBy::any(),
@@ -1318,7 +1514,7 @@ mod tests {
             assert_eq_input_ref!(&exprs[1], 1);
 
             assert_eq!(agg_calls.len(), 1);
-            assert_eq!(agg_calls[0].agg_kind, AggKind::Min);
+            assert_eq!(agg_calls[0].agg_kind, PbAggKind::Min.into());
             assert_eq!(input_ref_to_column_indices(&agg_calls[0].inputs), vec![1]);
             assert_eq!(group_key, vec![0].into());
         }
@@ -1335,14 +1531,13 @@ mod tests {
 
         let values = LogicalValues::new(vec![], Schema { fields }, ctx);
         let agg_call = PlanAggCall {
-            agg_kind: AggKind::Min,
+            agg_kind: PbAggKind::Min.into(),
             return_type: ty.clone(),
             inputs: vec![InputRef::new(2, ty.clone())],
             distinct: false,
             order_by: vec![],
             filter: Condition::true_cond(),
             direct_args: vec![],
-            user_defined: None,
         };
         Agg::new(vec![agg_call], vec![1].into(), values.into()).into()
     }
@@ -1376,7 +1571,7 @@ mod tests {
 
         assert_eq!(agg_new.agg_calls().len(), 1);
         let agg_call_new = agg_new.agg_calls()[0].clone();
-        assert_eq!(agg_call_new.agg_kind, AggKind::Min);
+        assert_eq!(agg_call_new.agg_kind, PbAggKind::Min.into());
         assert_eq!(input_ref_to_column_indices(&agg_call_new.inputs), vec![1]);
         assert_eq!(agg_call_new.return_type, ty);
 
@@ -1419,7 +1614,7 @@ mod tests {
 
         assert_eq!(agg_new.agg_calls().len(), 1);
         let agg_call_new = agg_new.agg_calls()[0].clone();
-        assert_eq!(agg_call_new.agg_kind, AggKind::Min);
+        assert_eq!(agg_call_new.agg_kind, PbAggKind::Min.into());
         assert_eq!(input_ref_to_column_indices(&agg_call_new.inputs), vec![1]);
         assert_eq!(agg_call_new.return_type, ty);
 
@@ -1456,14 +1651,13 @@ mod tests {
             ctx,
         );
         let agg_call = PlanAggCall {
-            agg_kind: AggKind::Min,
+            agg_kind: PbAggKind::Min.into(),
             return_type: ty.clone(),
             inputs: vec![InputRef::new(2, ty.clone())],
             distinct: false,
             order_by: vec![],
             filter: Condition::true_cond(),
             direct_args: vec![],
-            user_defined: None,
         };
         let agg: PlanRef = Agg::new(vec![agg_call], vec![1].into(), values.into()).into();
 
@@ -1482,7 +1676,7 @@ mod tests {
 
         assert_eq!(agg_new.agg_calls().len(), 1);
         let agg_call_new = agg_new.agg_calls()[0].clone();
-        assert_eq!(agg_call_new.agg_kind, AggKind::Min);
+        assert_eq!(agg_call_new.agg_kind, PbAggKind::Min.into());
         assert_eq!(input_ref_to_column_indices(&agg_call_new.inputs), vec![1]);
         assert_eq!(agg_call_new.return_type, ty);
 
@@ -1521,24 +1715,22 @@ mod tests {
 
         let agg_calls = vec![
             PlanAggCall {
-                agg_kind: AggKind::Min,
+                agg_kind: PbAggKind::Min.into(),
                 return_type: ty.clone(),
                 inputs: vec![InputRef::new(2, ty.clone())],
                 distinct: false,
                 order_by: vec![],
                 filter: Condition::true_cond(),
                 direct_args: vec![],
-                user_defined: None,
             },
             PlanAggCall {
-                agg_kind: AggKind::Max,
+                agg_kind: PbAggKind::Max.into(),
                 return_type: ty.clone(),
                 inputs: vec![InputRef::new(1, ty.clone())],
                 distinct: false,
                 order_by: vec![],
                 filter: Condition::true_cond(),
                 direct_args: vec![],
-                user_defined: None,
             },
         ];
         let agg: PlanRef = Agg::new(agg_calls, vec![1, 2].into(), values.into()).into();
@@ -1558,7 +1750,7 @@ mod tests {
 
         assert_eq!(agg_new.agg_calls().len(), 1);
         let agg_call_new = agg_new.agg_calls()[0].clone();
-        assert_eq!(agg_call_new.agg_kind, AggKind::Max);
+        assert_eq!(agg_call_new.agg_kind, PbAggKind::Max.into());
         assert_eq!(input_ref_to_column_indices(&agg_call_new.inputs), vec![0]);
         assert_eq!(agg_call_new.return_type, ty);
 
diff --git a/src/frontend/src/optimizer/plan_node/logical_file_scan.rs b/src/frontend/src/optimizer/plan_node/logical_file_scan.rs
new file mode 100644
index 0000000000000..abe8e40a8224f
--- /dev/null
+++ b/src/frontend/src/optimizer/plan_node/logical_file_scan.rs
@@ -0,0 +1,118 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use pretty_xmlish::XmlNode;
+use risingwave_common::bail;
+use risingwave_common::catalog::Schema;
+
+use super::generic::GenericPlanRef;
+use super::utils::{childless_record, Distill};
+use super::{
+    generic, BatchFileScan, ColPrunable, ExprRewritable, Logical, LogicalProject, PlanBase,
+    PlanRef, PredicatePushdown, ToBatch, ToStream,
+};
+use crate::error::Result;
+use crate::optimizer::plan_node::expr_visitable::ExprVisitable;
+use crate::optimizer::plan_node::utils::column_names_pretty;
+use crate::optimizer::plan_node::{
+    ColumnPruningContext, LogicalFilter, PredicatePushdownContext, RewriteStreamContext,
+    ToStreamContext,
+};
+use crate::utils::{ColIndexMapping, Condition};
+use crate::OptimizerContextRef;
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct LogicalFileScan {
+    pub base: PlanBase<Logical>,
+    pub core: generic::FileScan,
+}
+
+impl LogicalFileScan {
+    pub fn new(
+        ctx: OptimizerContextRef,
+        schema: Schema,
+        file_format: String,
+        storage_type: String,
+        s3_region: String,
+        s3_access_key: String,
+        s3_secret_key: String,
+        file_location: Vec<String>,
+    ) -> Self {
+        assert!("parquet".eq_ignore_ascii_case(&file_format));
+        assert!("s3".eq_ignore_ascii_case(&storage_type));
+
+        let core = generic::FileScan {
+            schema,
+            file_format: generic::FileFormat::Parquet,
+            storage_type: generic::StorageType::S3,
+            s3_region,
+            s3_access_key,
+            s3_secret_key,
+            file_location,
+            ctx,
+        };
+
+        let base = PlanBase::new_logical_with_core(&core);
+
+        LogicalFileScan { base, core }
+    }
+}
+
+impl_plan_tree_node_for_leaf! {LogicalFileScan}
+impl Distill for LogicalFileScan {
+    fn distill<'a>(&self) -> XmlNode<'a> {
+        let fields = vec![("columns", column_names_pretty(self.schema()))];
+        childless_record("LogicalFileScan", fields)
+    }
+}
+
+impl ColPrunable for LogicalFileScan {
+    fn prune_col(&self, required_cols: &[usize], _ctx: &mut ColumnPruningContext) -> PlanRef {
+        LogicalProject::with_out_col_idx(self.clone().into(), required_cols.iter().cloned()).into()
+    }
+}
+
+impl ExprRewritable for LogicalFileScan {}
+
+impl ExprVisitable for LogicalFileScan {}
+
+impl PredicatePushdown for LogicalFileScan {
+    fn predicate_pushdown(
+        &self,
+        predicate: Condition,
+        _ctx: &mut PredicatePushdownContext,
+    ) -> PlanRef {
+        // No pushdown.
+        LogicalFilter::create(self.clone().into(), predicate)
+    }
+}
+
+impl ToBatch for LogicalFileScan {
+    fn to_batch(&self) -> Result<PlanRef> {
+        Ok(BatchFileScan::new(self.core.clone()).into())
+    }
+}
+
+impl ToStream for LogicalFileScan {
+    fn to_stream(&self, _ctx: &mut ToStreamContext) -> Result<PlanRef> {
+        bail!("file_scan function is not supported in streaming mode")
+    }
+
+    fn logical_rewrite_for_stream(
+        &self,
+        _ctx: &mut RewriteStreamContext,
+    ) -> Result<(PlanRef, ColIndexMapping)> {
+        bail!("file_scan function is not supported in streaming mode")
+    }
+}
diff --git a/src/frontend/src/optimizer/plan_node/logical_filter.rs b/src/frontend/src/optimizer/plan_node/logical_filter.rs
index 04cc2cb12a689..25062ee0eebc7 100644
--- a/src/frontend/src/optimizer/plan_node/logical_filter.rs
+++ b/src/frontend/src/optimizer/plan_node/logical_filter.rs
@@ -197,24 +197,11 @@ impl ToStream for LogicalFilter {
         let new_input = self.input().to_stream(ctx)?;
 
         let predicate = self.predicate();
-        let has_now = predicate
-            .conjunctions
-            .iter()
-            .any(|cond| cond.count_nows() > 0);
-        if has_now {
-            if predicate
-                .conjunctions
-                .iter()
-                .any(|expr| expr.count_nows() > 0 && expr.as_now_comparison_cond().is_none())
-            {
-                bail!(
-                    "Conditions containing now must be of the form `input_expr cmp now() [+- const_expr]` or \
-                    `now() [+- const_expr] cmp input_expr`, where `input_expr` references a column \
-                    and contains no `now()`."
-                );
-            }
+        if predicate.conjunctions.iter().any(|cond| cond.has_now()) {
             bail!(
-                "All `now()` exprs were valid, but the condition must have at least one now expr as a lower bound."
+                "Conditions containing now must be in the form of `input_expr cmp now_expr` or \
+                `now_expr cmp input_expr`, where `input_expr` references a column and contains \
+                no `now()`, and `now_expr` is a non-decreasing expression contains `now()`."
             );
         }
         let mut new_logical = self.core.clone();
diff --git a/src/frontend/src/optimizer/plan_node/logical_join.rs b/src/frontend/src/optimizer/plan_node/logical_join.rs
index a8a832407ba68..2b64b5fd93ad5 100644
--- a/src/frontend/src/optimizer/plan_node/logical_join.rs
+++ b/src/frontend/src/optimizer/plan_node/logical_join.rs
@@ -418,6 +418,7 @@ impl LogicalJoin {
             .collect_vec();
 
         let new_scan_output_column_ids = new_scan.output_column_ids();
+        let as_of = new_scan.as_of.clone();
 
         // Construct a new logical join, because we have change its RHS.
         let new_logical_join = generic::Join::new(
@@ -435,6 +436,7 @@ impl LogicalJoin {
             new_scan_output_column_ids,
             lookup_prefix_len,
             false,
+            as_of,
         ))
     }
 
diff --git a/src/frontend/src/optimizer/plan_node/logical_over_window.rs b/src/frontend/src/optimizer/plan_node/logical_over_window.rs
index bd38569397992..7a81b164fbafe 100644
--- a/src/frontend/src/optimizer/plan_node/logical_over_window.rs
+++ b/src/frontend/src/optimizer/plan_node/logical_over_window.rs
@@ -17,7 +17,7 @@ use itertools::Itertools;
 use risingwave_common::types::DataType;
 use risingwave_common::util::sort_util::{ColumnOrder, OrderType};
 use risingwave_common::{bail_not_implemented, not_implemented};
-use risingwave_expr::aggregate::AggKind;
+use risingwave_expr::aggregate::{AggKind, PbAggKind};
 use risingwave_expr::window_function::{Frame, FrameBound, WindowFuncKind};
 
 use super::generic::{GenericPlanRef, OverWindow, PlanWindowFunction, ProjectBuilder};
@@ -105,17 +105,19 @@ impl<'a> LogicalOverWindowBuilder<'a> {
             window_func.frame,
         );
 
-        let new_expr = if let WindowFuncKind::Aggregate(agg_kind) = kind
+        let new_expr = if let WindowFuncKind::Aggregate(agg_kind) = &kind
             && matches!(
                 agg_kind,
-                AggKind::Avg
-                    | AggKind::StddevPop
-                    | AggKind::StddevSamp
-                    | AggKind::VarPop
-                    | AggKind::VarSamp
+                AggKind::Builtin(
+                    PbAggKind::Avg
+                        | PbAggKind::StddevPop
+                        | PbAggKind::StddevSamp
+                        | PbAggKind::VarPop
+                        | PbAggKind::VarSamp
+                )
             ) {
             let agg_call = AggCall::new(
-                agg_kind,
+                agg_kind.clone(),
                 args,
                 false,
                 order_by,
@@ -186,10 +188,15 @@ impl<'a> OverWindowProjectBuilder<'a> {
         &mut self,
         window_function: &WindowFunction,
     ) -> std::result::Result<(), ErrorCode> {
-        if let WindowFuncKind::Aggregate(agg_kind) = window_function.kind
+        if let WindowFuncKind::Aggregate(agg_kind) = &window_function.kind
             && matches!(
                 agg_kind,
-                AggKind::StddevPop | AggKind::StddevSamp | AggKind::VarPop | AggKind::VarSamp
+                AggKind::Builtin(
+                    PbAggKind::StddevPop
+                        | PbAggKind::StddevSamp
+                        | PbAggKind::VarPop
+                        | PbAggKind::VarSamp
+                )
             )
         {
             let input = window_function.args.iter().exactly_one().unwrap();
@@ -371,7 +378,10 @@ impl LogicalOverWindow {
                     )
                 };
 
-                (WindowFuncKind::Aggregate(AggKind::FirstValue), frame)
+                (
+                    WindowFuncKind::Aggregate(AggKind::Builtin(PbAggKind::FirstValue)),
+                    frame,
+                )
             }
             WindowFuncKind::Aggregate(_) => {
                 let frame = window_function.frame.unwrap_or({
diff --git a/src/frontend/src/optimizer/plan_node/logical_source.rs b/src/frontend/src/optimizer/plan_node/logical_source.rs
index 918db2919e626..50024b4274e77 100644
--- a/src/frontend/src/optimizer/plan_node/logical_source.rs
+++ b/src/frontend/src/optimizer/plan_node/logical_source.rs
@@ -44,7 +44,7 @@ use crate::optimizer::plan_node::{
     ToStreamContext,
 };
 use crate::optimizer::property::Distribution::HashShard;
-use crate::optimizer::property::{Distribution, Order, RequiredDist};
+use crate::optimizer::property::{Distribution, MonotonicityMap, Order, RequiredDist};
 use crate::utils::{ColIndexMapping, Condition, IndexRewriter};
 
 /// `LogicalSource` returns contents of a table or other equivalent object
@@ -229,6 +229,7 @@ impl LogicalSource {
                 true, // `list` will keep listing all objects, it must be append-only
                 false,
                 FixedBitSet::with_capacity(logical_source.column_catalog.len()),
+                MonotonicityMap::new(),
             ),
             core: logical_source,
         }
diff --git a/src/frontend/src/optimizer/plan_node/mod.rs b/src/frontend/src/optimizer/plan_node/mod.rs
index 71c4c44fac8ba..db1200de2a27a 100644
--- a/src/frontend/src/optimizer/plan_node/mod.rs
+++ b/src/frontend/src/optimizer/plan_node/mod.rs
@@ -49,7 +49,7 @@ use self::batch::BatchPlanRef;
 use self::generic::{GenericPlanRef, PhysicalPlanRef};
 use self::stream::StreamPlanRef;
 use self::utils::Distill;
-use super::property::{Distribution, FunctionalDependencySet, Order};
+use super::property::{Distribution, FunctionalDependencySet, MonotonicityMap, Order};
 use crate::error::{ErrorCode, Result};
 use crate::optimizer::ExpressionSimplifyRewriter;
 use crate::session::current::notice_to_user;
@@ -609,6 +609,10 @@ impl StreamPlanRef for PlanRef {
     fn watermark_columns(&self) -> &FixedBitSet {
         self.plan_base().watermark_columns()
     }
+
+    fn columns_monotonicity(&self) -> &MonotonicityMap {
+        self.plan_base().columns_monotonicity()
+    }
 }
 
 /// Allow access to all fields defined in [`BatchPlanRef`] for the type-erased plan node.
@@ -889,16 +893,19 @@ mod stream_exchange;
 mod stream_expand;
 mod stream_filter;
 mod stream_fs_fetch;
+mod stream_global_approx_percentile;
 mod stream_group_topn;
 mod stream_hash_agg;
 mod stream_hash_join;
 mod stream_hop_window;
+mod stream_local_approx_percentile;
 mod stream_materialize;
 mod stream_now;
 mod stream_over_window;
 mod stream_project;
 mod stream_project_set;
 mod stream_row_id_gen;
+mod stream_row_merge;
 mod stream_simple_agg;
 mod stream_sink;
 mod stream_sort;
@@ -910,9 +917,11 @@ mod stream_topn;
 mod stream_values;
 mod stream_watermark_filter;
 
+mod batch_file_scan;
 mod batch_iceberg_scan;
 mod batch_kafka_scan;
 mod derive;
+mod logical_file_scan;
 mod logical_iceberg_scan;
 mod stream_cdc_table_scan;
 mod stream_share;
@@ -923,6 +932,7 @@ pub mod utils;
 pub use batch_delete::BatchDelete;
 pub use batch_exchange::BatchExchange;
 pub use batch_expand::BatchExpand;
+pub use batch_file_scan::BatchFileScan;
 pub use batch_filter::BatchFilter;
 pub use batch_group_topn::BatchGroupTopN;
 pub use batch_hash_agg::BatchHashAgg;
@@ -959,6 +969,7 @@ pub use logical_dedup::LogicalDedup;
 pub use logical_delete::LogicalDelete;
 pub use logical_except::LogicalExcept;
 pub use logical_expand::LogicalExpand;
+pub use logical_file_scan::LogicalFileScan;
 pub use logical_filter::LogicalFilter;
 pub use logical_hop_window::LogicalHopWindow;
 pub use logical_iceberg_scan::LogicalIcebergScan;
@@ -994,16 +1005,19 @@ pub use stream_exchange::StreamExchange;
 pub use stream_expand::StreamExpand;
 pub use stream_filter::StreamFilter;
 pub use stream_fs_fetch::StreamFsFetch;
+pub use stream_global_approx_percentile::StreamGlobalApproxPercentile;
 pub use stream_group_topn::StreamGroupTopN;
 pub use stream_hash_agg::StreamHashAgg;
 pub use stream_hash_join::StreamHashJoin;
 pub use stream_hop_window::StreamHopWindow;
+pub use stream_local_approx_percentile::StreamLocalApproxPercentile;
 pub use stream_materialize::StreamMaterialize;
 pub use stream_now::StreamNow;
 pub use stream_over_window::StreamOverWindow;
 pub use stream_project::StreamProject;
 pub use stream_project_set::StreamProjectSet;
 pub use stream_row_id_gen::StreamRowIdGen;
+pub use stream_row_merge::StreamRowMerge;
 pub use stream_share::StreamShare;
 pub use stream_simple_agg::StreamSimpleAgg;
 pub use stream_sink::{IcebergPartitionInfo, PartitionComputeInfo, StreamSink};
@@ -1076,6 +1090,7 @@ macro_rules! for_all_plan_nodes {
             , { Logical, RecursiveUnion }
             , { Logical, CteRef }
             , { Logical, ChangeLog }
+            , { Logical, FileScan }
             , { Batch, SimpleAgg }
             , { Batch, HashAgg }
             , { Batch, SortAgg }
@@ -1106,6 +1121,7 @@ macro_rules! for_all_plan_nodes {
             , { Batch, MaxOneRow }
             , { Batch, KafkaScan }
             , { Batch, IcebergScan }
+            , { Batch, FileScan }
             , { Stream, Project }
             , { Stream, Filter }
             , { Stream, TableScan }
@@ -1140,6 +1156,9 @@ macro_rules! for_all_plan_nodes {
             , { Stream, OverWindow }
             , { Stream, FsFetch }
             , { Stream, ChangeLog }
+            , { Stream, GlobalApproxPercentile }
+            , { Stream, LocalApproxPercentile }
+            , { Stream, RowMerge }
         }
     };
 }
@@ -1182,6 +1201,7 @@ macro_rules! for_logical_plan_nodes {
             , { Logical, RecursiveUnion }
             , { Logical, CteRef }
             , { Logical, ChangeLog }
+            , { Logical, FileScan }
         }
     };
 }
@@ -1221,6 +1241,7 @@ macro_rules! for_batch_plan_nodes {
             , { Batch, MaxOneRow }
             , { Batch, KafkaScan }
             , { Batch, IcebergScan }
+            , { Batch, FileScan }
         }
     };
 }
@@ -1264,6 +1285,9 @@ macro_rules! for_stream_plan_nodes {
             , { Stream, OverWindow }
             , { Stream, FsFetch }
             , { Stream, ChangeLog }
+            , { Stream, GlobalApproxPercentile }
+            , { Stream, LocalApproxPercentile }
+            , { Stream, RowMerge }
         }
     };
 }
diff --git a/src/frontend/src/optimizer/plan_node/plan_base.rs b/src/frontend/src/optimizer/plan_node/plan_base.rs
index 12fba475241c5..02c85858967f8 100644
--- a/src/frontend/src/optimizer/plan_node/plan_base.rs
+++ b/src/frontend/src/optimizer/plan_node/plan_base.rs
@@ -59,6 +59,8 @@ pub struct StreamExtra {
     /// The watermark column indices of the `PlanNode`'s output. There could be watermark output from
     /// this stream operator.
     watermark_columns: FixedBitSet,
+    /// The monotonicity of columns in the output.
+    columns_monotonicity: MonotonicityMap,
 }
 
 impl GetPhysicalCommon for StreamExtra {
@@ -168,6 +170,10 @@ impl stream::StreamPlanRef for PlanBase<Stream> {
     fn watermark_columns(&self) -> &FixedBitSet {
         &self.extra.watermark_columns
     }
+
+    fn columns_monotonicity(&self) -> &MonotonicityMap {
+        &self.extra.columns_monotonicity
+    }
 }
 
 impl batch::BatchPlanRef for PlanBase<Batch> {
@@ -222,6 +228,7 @@ impl PlanBase<Stream> {
         append_only: bool,
         emit_on_window_close: bool,
         watermark_columns: FixedBitSet,
+        columns_monotonicity: MonotonicityMap,
     ) -> Self {
         let id = ctx.next_plan_node_id();
         assert_eq!(watermark_columns.len(), schema.len());
@@ -236,6 +243,7 @@ impl PlanBase<Stream> {
                 append_only,
                 emit_on_window_close,
                 watermark_columns,
+                columns_monotonicity,
             },
         }
     }
@@ -246,6 +254,7 @@ impl PlanBase<Stream> {
         append_only: bool,
         emit_on_window_close: bool,
         watermark_columns: FixedBitSet,
+        columns_monotonicity: MonotonicityMap,
     ) -> Self {
         Self::new_stream(
             core.ctx(),
@@ -256,6 +265,7 @@ impl PlanBase<Stream> {
             append_only,
             emit_on_window_close,
             watermark_columns,
+            columns_monotonicity,
         )
     }
 }
@@ -383,6 +393,10 @@ impl<'a> PlanBaseRef<'a> {
         dispatch_plan_base!(self, [Stream], StreamPlanRef::watermark_columns)
     }
 
+    pub(super) fn columns_monotonicity(self) -> &'a MonotonicityMap {
+        dispatch_plan_base!(self, [Stream], StreamPlanRef::columns_monotonicity)
+    }
+
     pub(super) fn order(self) -> &'a Order {
         dispatch_plan_base!(self, [Batch], BatchPlanRef::order)
     }
@@ -428,6 +442,10 @@ impl StreamPlanRef for PlanBaseRef<'_> {
     fn watermark_columns(&self) -> &FixedBitSet {
         (*self).watermark_columns()
     }
+
+    fn columns_monotonicity(&self) -> &MonotonicityMap {
+        (*self).columns_monotonicity()
+    }
 }
 
 impl BatchPlanRef for PlanBaseRef<'_> {
diff --git a/src/frontend/src/optimizer/plan_node/stream.rs b/src/frontend/src/optimizer/plan_node/stream.rs
index 42a599ccd60b0..e2df99d13d9f4 100644
--- a/src/frontend/src/optimizer/plan_node/stream.rs
+++ b/src/frontend/src/optimizer/plan_node/stream.rs
@@ -15,6 +15,7 @@
 use fixedbitset::FixedBitSet;
 
 use super::generic::PhysicalPlanRef;
+use crate::optimizer::property::MonotonicityMap;
 
 /// A subtrait of [`PhysicalPlanRef`] for stream plans.
 ///
@@ -29,6 +30,7 @@ pub trait StreamPlanRef: PhysicalPlanRef {
     fn append_only(&self) -> bool;
     fn emit_on_window_close(&self) -> bool;
     fn watermark_columns(&self) -> &FixedBitSet;
+    fn columns_monotonicity(&self) -> &MonotonicityMap;
 }
 
 /// Prelude for stream plan nodes.
diff --git a/src/frontend/src/optimizer/plan_node/stream_cdc_table_scan.rs b/src/frontend/src/optimizer/plan_node/stream_cdc_table_scan.rs
index 9fe3347171453..406ebdc8bfae5 100644
--- a/src/frontend/src/optimizer/plan_node/stream_cdc_table_scan.rs
+++ b/src/frontend/src/optimizer/plan_node/stream_cdc_table_scan.rs
@@ -50,6 +50,7 @@ impl StreamCdcTableScan {
             core.append_only(),
             false,
             core.watermark_columns(),
+            core.columns_monotonicity(),
         );
         Self { base, core }
     }
@@ -255,7 +256,7 @@ impl StreamCdcTableScan {
             // The table desc used by backfill executor
             state_table: Some(catalog),
             cdc_table_desc: Some(self.core.cdc_table_desc.to_protobuf()),
-            rate_limit: self.base.ctx().overwrite_options().streaming_rate_limit,
+            rate_limit: self.base.ctx().overwrite_options().backfill_rate_limit,
             disable_backfill: options.disable_backfill,
             options: Some(options),
         });
diff --git a/src/frontend/src/optimizer/plan_node/stream_changelog.rs b/src/frontend/src/optimizer/plan_node/stream_changelog.rs
index d84aa8c7e0cdf..34bfdec281815 100644
--- a/src/frontend/src/optimizer/plan_node/stream_changelog.rs
+++ b/src/frontend/src/optimizer/plan_node/stream_changelog.rs
@@ -20,6 +20,7 @@ use super::stream::prelude::PhysicalPlanRef;
 use super::stream::StreamPlanRef;
 use super::utils::impl_distill_by_unit;
 use super::{generic, ExprRewritable, PlanBase, PlanTreeNodeUnary, Stream, StreamNode};
+use crate::optimizer::property::MonotonicityMap;
 use crate::stream_fragmenter::BuildFragmentGraphState;
 use crate::PlanRef;
 
@@ -33,19 +34,22 @@ impl StreamChangeLog {
     pub fn new(core: generic::ChangeLog<PlanRef>) -> Self {
         let input = core.input.clone();
         let dist = input.distribution().clone();
+        let input_len = input.schema().len();
         // Filter executor won't change the append-only behavior of the stream.
         let mut watermark_columns = input.watermark_columns().clone();
         if core.need_op {
-            watermark_columns.grow(input.watermark_columns().len() + 2);
+            watermark_columns.grow(input_len + 2);
         } else {
-            watermark_columns.grow(input.watermark_columns().len() + 1);
+            watermark_columns.grow(input_len + 1);
         }
         let base = PlanBase::new_stream_with_core(
             &core,
             dist,
-            input.append_only(),
+            // The changelog will convert all delete/update to insert, so it must be true here.
+            true,
             input.emit_on_window_close(),
             watermark_columns,
+            MonotonicityMap::new(), // TODO: derive monotonicity
         );
         StreamChangeLog { base, core }
     }
diff --git a/src/frontend/src/optimizer/plan_node/stream_dedup.rs b/src/frontend/src/optimizer/plan_node/stream_dedup.rs
index b31415c125507..d642d0f9e7ee0 100644
--- a/src/frontend/src/optimizer/plan_node/stream_dedup.rs
+++ b/src/frontend/src/optimizer/plan_node/stream_dedup.rs
@@ -44,6 +44,7 @@ impl StreamDedup {
             true,
             input.emit_on_window_close(),
             input.watermark_columns().clone(),
+            input.columns_monotonicity().clone(),
         );
         StreamDedup { base, core }
     }
diff --git a/src/frontend/src/optimizer/plan_node/stream_delta_join.rs b/src/frontend/src/optimizer/plan_node/stream_delta_join.rs
index 49257676bc004..f53d4331ae617 100644
--- a/src/frontend/src/optimizer/plan_node/stream_delta_join.rs
+++ b/src/frontend/src/optimizer/plan_node/stream_delta_join.rs
@@ -27,7 +27,7 @@ use crate::expr::{Expr, ExprRewriter, ExprVisitor};
 use crate::optimizer::plan_node::expr_visitable::ExprVisitable;
 use crate::optimizer::plan_node::utils::IndicesDisplay;
 use crate::optimizer::plan_node::{EqJoinPredicate, EqJoinPredicateDisplay, TryToStreamPb};
-use crate::optimizer::property::Distribution;
+use crate::optimizer::property::{Distribution, MonotonicityMap};
 use crate::scheduler::SchedulerResult;
 use crate::stream_fragmenter::BuildFragmentGraphState;
 use crate::utils::ColIndexMappingRewriteExt;
@@ -68,6 +68,7 @@ impl StreamDeltaJoin {
             let watermark_columns = from_left.bitand(&from_right);
             core.i2o_col_mapping().rewrite_bitset(&watermark_columns)
         };
+
         // TODO: derive from input
         let base = PlanBase::new_stream_with_core(
             &core,
@@ -75,6 +76,7 @@ impl StreamDeltaJoin {
             append_only,
             false, // TODO(rc): derive EOWC property from input
             watermark_columns,
+            MonotonicityMap::new(), // TODO: derive monotonicity
         );
 
         Self {
diff --git a/src/frontend/src/optimizer/plan_node/stream_dml.rs b/src/frontend/src/optimizer/plan_node/stream_dml.rs
index e777041718291..7b671efa24c23 100644
--- a/src/frontend/src/optimizer/plan_node/stream_dml.rs
+++ b/src/frontend/src/optimizer/plan_node/stream_dml.rs
@@ -21,6 +21,7 @@ use super::stream::prelude::*;
 use super::utils::{childless_record, Distill};
 use super::{ExprRewritable, PlanBase, PlanRef, PlanTreeNodeUnary, StreamNode};
 use crate::optimizer::plan_node::expr_visitable::ExprVisitable;
+use crate::optimizer::property::MonotonicityMap;
 use crate::stream_fragmenter::BuildFragmentGraphState;
 
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
@@ -41,6 +42,7 @@ impl StreamDml {
             append_only,
             false,                                            // TODO(rc): decide EOWC property
             FixedBitSet::with_capacity(input.schema().len()), // no watermark if dml is allowed
+            MonotonicityMap::new(),                           // TODO: derive monotonicity
         );
 
         Self {
diff --git a/src/frontend/src/optimizer/plan_node/stream_dynamic_filter.rs b/src/frontend/src/optimizer/plan_node/stream_dynamic_filter.rs
index a90cd4b77c669..02fb426905c7e 100644
--- a/src/frontend/src/optimizer/plan_node/stream_dynamic_filter.rs
+++ b/src/frontend/src/optimizer/plan_node/stream_dynamic_filter.rs
@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use fixedbitset::FixedBitSet;
 use pretty_xmlish::{Pretty, XmlNode};
 pub use risingwave_pb::expr::expr_node::Type as ExprType;
 use risingwave_pb::stream_plan::stream_node::NodeBody;
@@ -22,11 +23,11 @@ use super::stream::prelude::*;
 use super::utils::{
     childless_record, column_names_pretty, plan_node_name, watermark_pretty, Distill,
 };
-use super::{generic, ExprRewritable, PlanTreeNodeUnary};
-use crate::expr::{Expr, ExprImpl};
+use super::{generic, ExprRewritable};
+use crate::expr::Expr;
 use crate::optimizer::plan_node::expr_visitable::ExprVisitable;
 use crate::optimizer::plan_node::{PlanBase, PlanTreeNodeBinary, StreamNode};
-use crate::optimizer::property::Distribution;
+use crate::optimizer::property::MonotonicityMap;
 use crate::optimizer::PlanRef;
 use crate::stream_fragmenter::BuildFragmentGraphState;
 
@@ -35,74 +36,75 @@ pub struct StreamDynamicFilter {
     pub base: PlanBase<Stream>,
     core: generic::DynamicFilter<PlanRef>,
     cleaned_by_watermark: bool,
-    condition_always_relax: bool,
 }
 
 impl StreamDynamicFilter {
     pub fn new(core: DynamicFilter<PlanRef>) -> Self {
-        let watermark_columns = core.watermark_columns(core.right().watermark_columns()[0]);
-
-        // TODO(st1page): here we just check if RHS
-        // is a `StreamNow`. It will be generalized to more cases
-        // by introducing monotonically increasing property of the node in https://github.com/risingwavelabs/risingwave/pull/13984.
-        let right_monotonically_increasing = {
-            if let Some(e) = core.right().as_stream_exchange()
-                && *e.distribution() == Distribution::Broadcast
-            {
-                if e.input().as_stream_now().is_some() {
-                    true
-                } else if let Some(proj) = e.input().as_stream_project() {
-                    proj.input().as_stream_now().is_some()
-                } else {
-                    false
-                }
-            } else {
-                false
-            }
-        };
-        let condition_always_relax = right_monotonically_increasing
+        let right_non_decreasing = core.right().columns_monotonicity()[0].is_non_decreasing();
+        let condition_always_relax = right_non_decreasing
             && matches!(
                 core.comparator(),
                 ExprType::LessThan | ExprType::LessThanOrEqual
             );
 
-        let append_only = if condition_always_relax {
+        let out_append_only = if condition_always_relax {
             core.left().append_only()
         } else {
             false
         };
+
         let base = PlanBase::new_stream_with_core(
             &core,
             core.left().distribution().clone(),
-            append_only,
+            out_append_only,
             false, // TODO(rc): decide EOWC property
-            watermark_columns,
+            Self::derive_watermark_columns(&core),
+            MonotonicityMap::new(), // TODO: derive monotonicity
         );
         let cleaned_by_watermark = Self::cleaned_by_watermark(&core);
         Self {
             base,
             core,
             cleaned_by_watermark,
-            condition_always_relax,
         }
     }
 
-    pub fn left_index(&self) -> usize {
-        self.core.left_index()
+    fn derive_watermark_columns(core: &DynamicFilter<PlanRef>) -> FixedBitSet {
+        let mut res = FixedBitSet::with_capacity(core.left().schema().len());
+        let rhs_watermark_columns = core.right().watermark_columns();
+        if rhs_watermark_columns.contains(0) {
+            match core.comparator() {
+                // We can derive output watermark only if the output is supposed to be always >= rhs.
+                // While we have to keep in mind that, the propagation of watermark messages from
+                // the right input must be delayed until `Update`/`Delete`s are sent to downstream,
+                // otherwise, we will have watermark messages sent before the `Delete` of old rows.
+                ExprType::GreaterThan | ExprType::GreaterThanOrEqual => {
+                    res.set(core.left_index(), true)
+                }
+                _ => {}
+            }
+        }
+        res
     }
 
-    /// 1. Check the comparator.
-    /// 2. RHS input should only have 1 columns, which is the watermark column.
-    ///    We check that the watermark should be set.
-    pub fn cleaned_by_watermark(core: &DynamicFilter<PlanRef>) -> bool {
-        let expr = core.predicate();
-        if let Some(ExprImpl::FunctionCall(function_call)) = expr.as_expr_unless_true() {
-            match function_call.func_type() {
+    fn cleaned_by_watermark(core: &DynamicFilter<PlanRef>) -> bool {
+        let rhs_watermark_columns = core.right().watermark_columns();
+        if rhs_watermark_columns.contains(0) {
+            match core.comparator() {
                 ExprType::GreaterThan | ExprType::GreaterThanOrEqual => {
-                    let rhs_input = core.right();
-                    rhs_input.watermark_columns().contains(0)
+                    // For >= and >, watermark on rhs means there's no change that rows older than the watermark will
+                    // ever be `Insert`ed again. So, we can clean up the state table. In this case, future lhs inputs
+                    // that are less than the watermark can be safely ignored, and hence watermark can be propagated to
+                    // downstream. See `derive_watermark_columns`.
+                    true
+                }
+                ExprType::LessThan | ExprType::LessThanOrEqual => {
+                    // For <= and <, watermark on rhs means all rows older than the watermark should already be emitted,
+                    // and future lhs inputs should be directly passed to downstream without any state table operation.
+                    // So, the state table can be cleaned up.
+                    true
                 }
-                _ => false,
+                _ => unreachable!(),
             }
         } else {
             false
@@ -126,12 +128,6 @@ impl Distill for StreamDynamicFilter {
                 Pretty::display(&self.cleaned_by_watermark),
             ));
         }
-        if self.condition_always_relax {
-            vec.push((
-                "condition_always_relax",
-                Pretty::display(&self.condition_always_relax),
-            ));
-        }
         childless_record(
             plan_node_name!(
                 "StreamDynamicFilter",
@@ -174,12 +170,13 @@ impl StreamNode for StreamDynamicFilter {
         let right = self.right();
         let right_table = infer_right_internal_table_catalog(right.plan_base())
             .with_id(state.gen_table_id_wrapped());
+        #[allow(deprecated)]
         NodeBody::DynamicFilter(DynamicFilterNode {
             left_key: left_index as u32,
             condition,
             left_table: Some(left_table.to_internal_table_prost()),
             right_table: Some(right_table.to_internal_table_prost()),
-            condition_always_relax: self.condition_always_relax,
+            condition_always_relax: false, // deprecated
         })
     }
 }
diff --git a/src/frontend/src/optimizer/plan_node/stream_eowc_over_window.rs b/src/frontend/src/optimizer/plan_node/stream_eowc_over_window.rs
index 78cb0e3b9d605..4d134df37799b 100644
--- a/src/frontend/src/optimizer/plan_node/stream_eowc_over_window.rs
+++ b/src/frontend/src/optimizer/plan_node/stream_eowc_over_window.rs
@@ -23,6 +23,7 @@ use super::stream::prelude::*;
 use super::utils::{impl_distill_by_unit, TableCatalogBuilder};
 use super::{ExprRewritable, PlanBase, PlanRef, PlanTreeNodeUnary, StreamNode};
 use crate::optimizer::plan_node::expr_visitable::ExprVisitable;
+use crate::optimizer::property::MonotonicityMap;
 use crate::stream_fragmenter::BuildFragmentGraphState;
 use crate::TableCatalog;
 
@@ -58,6 +59,8 @@ impl StreamEowcOverWindow {
             true,
             true,
             watermark_columns,
+            // we cannot derive monotonicity for any column for the same reason as watermark columns
+            MonotonicityMap::new(),
         );
         StreamEowcOverWindow { base, core }
     }
diff --git a/src/frontend/src/optimizer/plan_node/stream_exchange.rs b/src/frontend/src/optimizer/plan_node/stream_exchange.rs
index 964accc0f69a4..d42f9a9392b41 100644
--- a/src/frontend/src/optimizer/plan_node/stream_exchange.rs
+++ b/src/frontend/src/optimizer/plan_node/stream_exchange.rs
@@ -20,7 +20,9 @@ use super::stream::prelude::*;
 use super::utils::{childless_record, plan_node_name, Distill};
 use super::{ExprRewritable, PlanBase, PlanRef, PlanTreeNodeUnary, StreamNode};
 use crate::optimizer::plan_node::expr_visitable::ExprVisitable;
-use crate::optimizer::property::{Distribution, DistributionDisplay};
+use crate::optimizer::property::{
+    Distribution, DistributionDisplay, MonotonicityMap, RequiredDist,
+};
 use crate::stream_fragmenter::BuildFragmentGraphState;
 
 /// `StreamExchange` imposes a particular distribution on its input
@@ -34,16 +36,24 @@ pub struct StreamExchange {
 
 impl StreamExchange {
     pub fn new(input: PlanRef, dist: Distribution) -> Self {
-        // Dispatch executor won't change the append-only behavior of the stream.
+        let columns_monotonicity = if input.distribution().satisfies(&RequiredDist::single()) {
+            // If the input is a singleton, the monotonicity will be preserved during shuffle
+            // since we use ordered channel/buffer when exchanging data.
+            input.columns_monotonicity().clone()
+        } else {
+            MonotonicityMap::new()
+        };
+        assert!(!input.schema().is_empty());
         let base = PlanBase::new_stream(
             input.ctx(),
             input.schema().clone(),
             input.stream_key().map(|v| v.to_vec()),
             input.functional_dependency().clone(),
             dist,
-            input.append_only(),
+            input.append_only(), // append-only property won't change
             input.emit_on_window_close(),
             input.watermark_columns().clone(),
+            columns_monotonicity,
         );
         StreamExchange {
             base,
@@ -54,16 +64,16 @@ impl StreamExchange {
 
     pub fn new_no_shuffle(input: PlanRef) -> Self {
         let ctx = input.ctx();
-        // Dispatch executor won't change the append-only behavior of the stream.
         let base = PlanBase::new_stream(
             ctx,
             input.schema().clone(),
             input.stream_key().map(|v| v.to_vec()),
             input.functional_dependency().clone(),
             input.distribution().clone(),
-            input.append_only(),
+            input.append_only(), // append-only property won't change
             input.emit_on_window_close(),
             input.watermark_columns().clone(),
+            input.columns_monotonicity().clone(),
         );
         StreamExchange {
             base,
diff --git a/src/frontend/src/optimizer/plan_node/stream_expand.rs b/src/frontend/src/optimizer/plan_node/stream_expand.rs
index 5eefede3469c0..fa0268a46fcf5 100644
--- a/src/frontend/src/optimizer/plan_node/stream_expand.rs
+++ b/src/frontend/src/optimizer/plan_node/stream_expand.rs
@@ -21,7 +21,7 @@ use super::stream::prelude::*;
 use super::utils::impl_distill_by_unit;
 use super::{generic, ExprRewritable, PlanBase, PlanRef, PlanTreeNodeUnary, StreamNode};
 use crate::optimizer::plan_node::expr_visitable::ExprVisitable;
-use crate::optimizer::property::Distribution;
+use crate::optimizer::property::{Distribution, MonotonicityMap};
 use crate::stream_fragmenter::BuildFragmentGraphState;
 
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
@@ -33,6 +33,7 @@ pub struct StreamExpand {
 impl StreamExpand {
     pub fn new(core: generic::Expand<PlanRef>) -> Self {
         let input = core.input.clone();
+        let input_len = input.schema().len();
 
         let dist = match input.distribution() {
             Distribution::Single => Distribution::Single,
@@ -43,12 +44,7 @@ impl StreamExpand {
         };
 
         let mut watermark_columns = FixedBitSet::with_capacity(core.output_len());
-        watermark_columns.extend(
-            input
-                .watermark_columns()
-                .ones()
-                .map(|idx| idx + input.schema().len()),
-        );
+        watermark_columns.extend(input.watermark_columns().ones().map(|idx| idx + input_len));
 
         let base = PlanBase::new_stream_with_core(
             &core,
@@ -56,6 +52,7 @@ impl StreamExpand {
             input.append_only(),
             input.emit_on_window_close(),
             watermark_columns,
+            MonotonicityMap::new(),
         );
         StreamExpand { base, core }
     }
diff --git a/src/frontend/src/optimizer/plan_node/stream_filter.rs b/src/frontend/src/optimizer/plan_node/stream_filter.rs
index 586b5ac8a84b9..0a3126ffe7180 100644
--- a/src/frontend/src/optimizer/plan_node/stream_filter.rs
+++ b/src/frontend/src/optimizer/plan_node/stream_filter.rs
@@ -42,6 +42,7 @@ impl StreamFilter {
             input.append_only(),
             input.emit_on_window_close(),
             input.watermark_columns().clone(),
+            input.columns_monotonicity().clone(),
         );
         StreamFilter { base, core }
     }
diff --git a/src/frontend/src/optimizer/plan_node/stream_fs_fetch.rs b/src/frontend/src/optimizer/plan_node/stream_fs_fetch.rs
index 43c14686e2a22..3f8c89d7def1a 100644
--- a/src/frontend/src/optimizer/plan_node/stream_fs_fetch.rs
+++ b/src/frontend/src/optimizer/plan_node/stream_fs_fetch.rs
@@ -26,7 +26,7 @@ use crate::catalog::source_catalog::SourceCatalog;
 use crate::optimizer::plan_node::expr_visitable::ExprVisitable;
 use crate::optimizer::plan_node::utils::{childless_record, Distill};
 use crate::optimizer::plan_node::{generic, ExprRewritable, StreamNode};
-use crate::optimizer::property::Distribution;
+use crate::optimizer::property::{Distribution, MonotonicityMap};
 use crate::stream_fragmenter::BuildFragmentGraphState;
 
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
@@ -55,6 +55,7 @@ impl StreamFsFetch {
             source.catalog.as_ref().map_or(true, |s| s.append_only),
             false,
             FixedBitSet::with_capacity(source.column_catalog.len()),
+            MonotonicityMap::new(), // TODO: derive monotonicity
         );
 
         Self {
@@ -97,25 +98,30 @@ impl StreamNode for StreamFsFetch {
     fn to_stream_prost_body(&self, state: &mut BuildFragmentGraphState) -> NodeBody {
         // `StreamFsFetch` is same as source in proto def, so the following code is the same as `StreamSource`
         let source_catalog = self.source_catalog();
-        let source_inner = source_catalog.map(|source_catalog| PbStreamFsFetch {
-            source_id: source_catalog.id,
-            source_name: source_catalog.name.clone(),
-            state_table: Some(
-                generic::Source::infer_internal_table_catalog(true)
-                    .with_id(state.gen_table_id_wrapped())
-                    .to_internal_table_prost(),
-            ),
-            info: Some(source_catalog.info.clone()),
-            row_id_index: self.core.row_id_index.map(|index| index as _),
-            columns: self
-                .core
-                .column_catalog
-                .iter()
-                .map(|c| c.to_protobuf())
-                .collect_vec(),
-            with_properties: source_catalog.with_properties.clone().into_iter().collect(),
-            rate_limit: self.base.ctx().overwrite_options().streaming_rate_limit,
-            secret_refs: Default::default(),
+
+        let source_inner = source_catalog.map(|source_catalog| {
+            let (with_properties, secret_refs) =
+                source_catalog.with_properties.clone().into_parts();
+            PbStreamFsFetch {
+                source_id: source_catalog.id,
+                source_name: source_catalog.name.clone(),
+                state_table: Some(
+                    generic::Source::infer_internal_table_catalog(true)
+                        .with_id(state.gen_table_id_wrapped())
+                        .to_internal_table_prost(),
+                ),
+                info: Some(source_catalog.info.clone()),
+                row_id_index: self.core.row_id_index.map(|index| index as _),
+                columns: self
+                    .core
+                    .column_catalog
+                    .iter()
+                    .map(|c| c.to_protobuf())
+                    .collect_vec(),
+                with_properties,
+                rate_limit: self.base.ctx().overwrite_options().source_rate_limit,
+                secret_refs,
+            }
         });
         NodeBody::StreamFsFetch(StreamFsFetchNode {
             node_inner: source_inner,
diff --git a/src/frontend/src/optimizer/plan_node/stream_global_approx_percentile.rs b/src/frontend/src/optimizer/plan_node/stream_global_approx_percentile.rs
new file mode 100644
index 0000000000000..2edd61a728acc
--- /dev/null
+++ b/src/frontend/src/optimizer/plan_node/stream_global_approx_percentile.rs
@@ -0,0 +1,151 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use fixedbitset::FixedBitSet;
+use pretty_xmlish::{Pretty, XmlNode};
+use risingwave_common::catalog::{Field, Schema};
+use risingwave_common::types::DataType;
+use risingwave_common::util::sort_util::OrderType;
+use risingwave_pb::stream_plan::stream_node::PbNodeBody;
+use risingwave_pb::stream_plan::GlobalApproxPercentileNode;
+
+use crate::expr::{ExprRewriter, ExprVisitor, Literal};
+use crate::optimizer::plan_node::expr_visitable::ExprVisitable;
+use crate::optimizer::plan_node::generic::GenericPlanRef;
+use crate::optimizer::plan_node::stream::StreamPlanRef;
+use crate::optimizer::plan_node::utils::{childless_record, Distill, TableCatalogBuilder};
+use crate::optimizer::plan_node::{
+    ExprRewritable, PlanAggCall, PlanBase, PlanTreeNodeUnary, Stream, StreamNode,
+};
+use crate::optimizer::property::Distribution;
+use crate::stream_fragmenter::BuildFragmentGraphState;
+use crate::PlanRef;
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct StreamGlobalApproxPercentile {
+    pub base: PlanBase<Stream>,
+    input: PlanRef,
+    /// Quantile
+    quantile: Literal,
+    /// Used to compute the exponent bucket base.
+    relative_error: Literal,
+}
+
+impl StreamGlobalApproxPercentile {
+    pub fn new(input: PlanRef, approx_percentile_agg_call: &PlanAggCall) -> Self {
+        let schema = Schema::new(vec![Field::with_name(
+            DataType::Float64,
+            "approx_percentile",
+        )]);
+        let watermark_columns = FixedBitSet::with_capacity(1);
+        let base = PlanBase::new_stream(
+            input.ctx(),
+            schema,
+            Some(vec![]),
+            input.functional_dependency().clone(),
+            Distribution::Single,
+            input.append_only(),
+            input.emit_on_window_close(),
+            watermark_columns,
+            input.columns_monotonicity().clone(),
+        );
+        Self {
+            base,
+            input,
+            quantile: approx_percentile_agg_call.direct_args[0].clone(),
+            relative_error: approx_percentile_agg_call.direct_args[1].clone(),
+        }
+    }
+}
+
+impl Distill for StreamGlobalApproxPercentile {
+    fn distill<'a>(&self) -> XmlNode<'a> {
+        let out = vec![
+            ("quantile", Pretty::debug(&self.quantile)),
+            ("relative_error", Pretty::debug(&self.relative_error)),
+        ];
+        childless_record("StreamGlobalApproxPercentile", out)
+    }
+}
+
+impl PlanTreeNodeUnary for StreamGlobalApproxPercentile {
+    fn input(&self) -> PlanRef {
+        self.input.clone()
+    }
+
+    fn clone_with_input(&self, input: PlanRef) -> Self {
+        Self {
+            base: self.base.clone(),
+            input,
+            quantile: self.quantile.clone(),
+            relative_error: self.relative_error.clone(),
+        }
+    }
+}
+
+impl_plan_tree_node_for_unary! {StreamGlobalApproxPercentile}
+
+impl StreamNode for StreamGlobalApproxPercentile {
+    fn to_stream_prost_body(&self, state: &mut BuildFragmentGraphState) -> PbNodeBody {
+        let relative_error = self.relative_error.get_data().as_ref().unwrap();
+        let relative_error = relative_error.as_float64().into_inner();
+        let base = (1.0 + relative_error) / (1.0 - relative_error);
+        let quantile = self.quantile.get_data().as_ref().unwrap();
+        let quantile = quantile.as_float64().into_inner();
+
+        // setup table: bucket_id->count
+        let mut bucket_table_builder = TableCatalogBuilder::default();
+        bucket_table_builder.add_column(&Field::with_name(DataType::Int16, "sign"));
+        bucket_table_builder.add_column(&Field::with_name(DataType::Int32, "bucket_id"));
+        bucket_table_builder.add_column(&Field::with_name(DataType::Int64, "count"));
+        bucket_table_builder.add_order_column(0, OrderType::ascending()); // sign
+        bucket_table_builder.add_order_column(1, OrderType::ascending()); // bucket_id
+
+        // setup table: total_count
+        let mut count_table_builder = TableCatalogBuilder::default();
+        count_table_builder.add_column(&Field::with_name(DataType::Int64, "total_count"));
+
+        let body = GlobalApproxPercentileNode {
+            base,
+            quantile,
+            bucket_state_table: Some(
+                bucket_table_builder
+                    .build(vec![], 0)
+                    .with_id(state.gen_table_id_wrapped())
+                    .to_internal_table_prost(),
+            ),
+            count_state_table: Some(
+                count_table_builder
+                    .build(vec![], 0)
+                    .with_id(state.gen_table_id_wrapped())
+                    .to_internal_table_prost(),
+            ),
+        };
+        PbNodeBody::GlobalApproxPercentile(body)
+    }
+}
+
+impl ExprRewritable for StreamGlobalApproxPercentile {
+    fn has_rewritable_expr(&self) -> bool {
+        false
+    }
+
+    fn rewrite_exprs(&self, _rewriter: &mut dyn ExprRewriter) -> PlanRef {
+        unimplemented!()
+    }
+}
+
+impl ExprVisitable for StreamGlobalApproxPercentile {
+    fn visit_exprs(&self, _v: &mut dyn ExprVisitor) {}
+}
diff --git a/src/frontend/src/optimizer/plan_node/stream_group_topn.rs b/src/frontend/src/optimizer/plan_node/stream_group_topn.rs
index 8500e24b0fd9f..b9230270e634e 100644
--- a/src/frontend/src/optimizer/plan_node/stream_group_topn.rs
+++ b/src/frontend/src/optimizer/plan_node/stream_group_topn.rs
@@ -22,7 +22,7 @@ use super::utils::{plan_node_name, watermark_pretty, Distill};
 use super::{generic, ExprRewritable, PlanBase, PlanTreeNodeUnary, StreamNode};
 use crate::optimizer::plan_node::expr_visitable::ExprVisitable;
 use crate::optimizer::plan_node::generic::GenericPlanNode;
-use crate::optimizer::property::Order;
+use crate::optimizer::property::{MonotonicityMap, Order};
 use crate::stream_fragmenter::BuildFragmentGraphState;
 use crate::PlanRef;
 
@@ -42,6 +42,8 @@ impl StreamGroupTopN {
         let input = &core.input;
         let schema = input.schema().clone();
 
+        // FIXME(rc): Actually only watermark messages on the first group-by column are propagated
+        // acccoring to the current GroupTopN implementation. This should be fixed.
         let watermark_columns = if input.append_only() {
             input.watermark_columns().clone()
         } else {
@@ -77,6 +79,7 @@ impl StreamGroupTopN {
             // TODO: https://github.com/risingwavelabs/risingwave/issues/8348
             false,
             watermark_columns,
+            MonotonicityMap::new(), // TODO: derive monotonicity
         );
         StreamGroupTopN {
             base,
diff --git a/src/frontend/src/optimizer/plan_node/stream_hash_agg.rs b/src/frontend/src/optimizer/plan_node/stream_hash_agg.rs
index eb69d5c259bb6..2dfad775ecc6e 100644
--- a/src/frontend/src/optimizer/plan_node/stream_hash_agg.rs
+++ b/src/frontend/src/optimizer/plan_node/stream_hash_agg.rs
@@ -24,6 +24,7 @@ use super::{ExprRewritable, PlanBase, PlanRef, PlanTreeNodeUnary, StreamNode};
 use crate::error::{ErrorCode, Result};
 use crate::expr::{ExprRewriter, ExprVisitor};
 use crate::optimizer::plan_node::expr_visitable::ExprVisitable;
+use crate::optimizer::property::MonotonicityMap;
 use crate::stream_fragmenter::BuildFragmentGraphState;
 use crate::utils::{ColIndexMapping, ColIndexMappingRewriteExt, IndexSet};
 
@@ -93,6 +94,7 @@ impl StreamHashAgg {
             emit_on_window_close, // in EOWC mode, we produce append only output
             emit_on_window_close,
             watermark_columns,
+            MonotonicityMap::new(), // TODO: derive monotonicity
         );
         StreamHashAgg {
             base,
diff --git a/src/frontend/src/optimizer/plan_node/stream_hash_join.rs b/src/frontend/src/optimizer/plan_node/stream_hash_join.rs
index b803fccef7b07..cbce1e1caf45a 100644
--- a/src/frontend/src/optimizer/plan_node/stream_hash_join.rs
+++ b/src/frontend/src/optimizer/plan_node/stream_hash_join.rs
@@ -30,7 +30,7 @@ use crate::expr::{Expr, ExprDisplay, ExprRewriter, ExprVisitor, InequalityInputP
 use crate::optimizer::plan_node::expr_visitable::ExprVisitable;
 use crate::optimizer::plan_node::utils::IndicesDisplay;
 use crate::optimizer::plan_node::{EqJoinPredicate, EqJoinPredicateDisplay};
-use crate::optimizer::property::Distribution;
+use crate::optimizer::property::{Distribution, MonotonicityMap};
 use crate::stream_fragmenter::BuildFragmentGraphState;
 use crate::utils::ColIndexMappingRewriteExt;
 
@@ -196,6 +196,7 @@ impl StreamHashJoin {
             append_only,
             false, // TODO(rc): derive EOWC property from input
             watermark_columns,
+            MonotonicityMap::new(), // TODO: derive monotonicity
         );
 
         Self {
diff --git a/src/frontend/src/optimizer/plan_node/stream_hop_window.rs b/src/frontend/src/optimizer/plan_node/stream_hop_window.rs
index 0cdddf77ed0e5..4a50387c50be0 100644
--- a/src/frontend/src/optimizer/plan_node/stream_hop_window.rs
+++ b/src/frontend/src/optimizer/plan_node/stream_hop_window.rs
@@ -13,7 +13,6 @@
 // limitations under the License.
 
 use pretty_xmlish::XmlNode;
-use risingwave_common::util::column_index_mapping::ColIndexMapping;
 use risingwave_pb::stream_plan::stream_node::PbNodeBody;
 use risingwave_pb::stream_plan::HopWindowNode;
 
@@ -22,6 +21,7 @@ use super::utils::{childless_record, watermark_pretty, Distill};
 use super::{generic, ExprRewritable, PlanBase, PlanRef, PlanTreeNodeUnary, StreamNode};
 use crate::expr::{Expr, ExprImpl, ExprRewriter, ExprVisitor};
 use crate::optimizer::plan_node::expr_visitable::ExprVisitable;
+use crate::optimizer::property::MonotonicityMap;
 use crate::stream_fragmenter::BuildFragmentGraphState;
 use crate::utils::ColIndexMappingRewriteExt;
 
@@ -41,29 +41,29 @@ impl StreamHopWindow {
         window_end_exprs: Vec<ExprImpl>,
     ) -> Self {
         let input = core.input.clone();
-        let i2o = core.i2o_col_mapping();
-        let dist = i2o.rewrite_provided_distribution(input.distribution());
+        let dist = core
+            .i2o_col_mapping()
+            .rewrite_provided_distribution(input.distribution());
 
-        let mut watermark_columns = input.watermark_columns().clone();
+        let input2internal = core.input2internal_col_mapping();
+        let internal2output = core.internal2output_col_mapping();
+
+        let mut watermark_columns = input2internal.rewrite_bitset(input.watermark_columns());
         watermark_columns.grow(core.internal_column_num());
 
-        if watermark_columns.contains(core.time_col.index) {
+        if input.watermark_columns().contains(core.time_col.index) {
             // Watermark on `time_col` indicates watermark on both `window_start` and `window_end`.
             watermark_columns.insert(core.internal_window_start_col_idx());
             watermark_columns.insert(core.internal_window_end_col_idx());
         }
-        let watermark_columns = ColIndexMapping::with_remaining_columns(
-            &core.output_indices,
-            core.internal_column_num(),
-        )
-        .rewrite_bitset(&watermark_columns);
 
         let base = PlanBase::new_stream_with_core(
             &core,
             dist,
             input.append_only(),
             input.emit_on_window_close(),
-            watermark_columns,
+            internal2output.rewrite_bitset(&watermark_columns),
+            MonotonicityMap::new(), /* hop window start/end jumps, so monotonicity is not propagated */
         );
         Self {
             base,
diff --git a/src/frontend/src/optimizer/plan_node/stream_local_approx_percentile.rs b/src/frontend/src/optimizer/plan_node/stream_local_approx_percentile.rs
new file mode 100644
index 0000000000000..20d27e41f8598
--- /dev/null
+++ b/src/frontend/src/optimizer/plan_node/stream_local_approx_percentile.rs
@@ -0,0 +1,137 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use fixedbitset::FixedBitSet;
+use pretty_xmlish::{Pretty, XmlNode};
+use risingwave_common::catalog::{Field, Schema};
+use risingwave_common::types::DataType;
+use risingwave_pb::stream_plan::stream_node::PbNodeBody;
+use risingwave_pb::stream_plan::LocalApproxPercentileNode;
+
+use crate::expr::{ExprRewriter, ExprVisitor, InputRef, InputRefDisplay, Literal};
+use crate::optimizer::plan_node::expr_visitable::ExprVisitable;
+use crate::optimizer::plan_node::generic::{GenericPlanRef, PhysicalPlanRef};
+use crate::optimizer::plan_node::stream::StreamPlanRef;
+use crate::optimizer::plan_node::utils::{childless_record, watermark_pretty, Distill};
+use crate::optimizer::plan_node::{
+    ExprRewritable, PlanAggCall, PlanBase, PlanTreeNodeUnary, Stream, StreamNode,
+};
+use crate::stream_fragmenter::BuildFragmentGraphState;
+use crate::PlanRef;
+
+// Does not contain `core` because no other plan nodes share
+// common fields and schema, even GlobalApproxPercentile.
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct StreamLocalApproxPercentile {
+    pub base: PlanBase<Stream>,
+    input: PlanRef,
+    quantile: Literal,
+    relative_error: Literal,
+    percentile_col: InputRef,
+}
+
+impl StreamLocalApproxPercentile {
+    pub fn new(input: PlanRef, approx_percentile_agg_call: &PlanAggCall) -> Self {
+        let schema = Schema::new(vec![
+            Field::with_name(DataType::Int16, "sign"),
+            Field::with_name(DataType::Int32, "bucket_id"),
+            Field::with_name(DataType::Int32, "count"),
+        ]);
+        // FIXME(kwannoel): How does watermark work with FixedBitSet
+        let watermark_columns = FixedBitSet::with_capacity(3);
+        let base = PlanBase::new_stream(
+            input.ctx(),
+            schema,
+            input.stream_key().map(|k| k.to_vec()),
+            input.functional_dependency().clone(),
+            input.distribution().clone(),
+            input.append_only(),
+            input.emit_on_window_close(),
+            watermark_columns,
+            input.columns_monotonicity().clone(),
+        );
+        Self {
+            base,
+            input,
+            quantile: approx_percentile_agg_call.direct_args[0].clone(),
+            relative_error: approx_percentile_agg_call.direct_args[1].clone(),
+            percentile_col: approx_percentile_agg_call.inputs[0].clone(),
+        }
+    }
+}
+
+impl Distill for StreamLocalApproxPercentile {
+    fn distill<'a>(&self) -> XmlNode<'a> {
+        let mut out = Vec::with_capacity(5);
+        out.push((
+            "percentile_col",
+            Pretty::display(&InputRefDisplay {
+                input_ref: &self.percentile_col,
+                input_schema: self.input.schema(),
+            }),
+        ));
+        out.push(("quantile", Pretty::debug(&self.quantile)));
+        out.push(("relative_error", Pretty::debug(&self.relative_error)));
+        if let Some(ow) = watermark_pretty(self.base.watermark_columns(), self.schema()) {
+            out.push(("output_watermarks", ow));
+        }
+        childless_record("StreamLocalApproxPercentile", out)
+    }
+}
+
+impl PlanTreeNodeUnary for StreamLocalApproxPercentile {
+    fn input(&self) -> PlanRef {
+        self.input.clone()
+    }
+
+    fn clone_with_input(&self, input: PlanRef) -> Self {
+        Self {
+            base: self.base.clone(),
+            input,
+            quantile: self.quantile.clone(),
+            relative_error: self.relative_error.clone(),
+            percentile_col: self.percentile_col.clone(),
+        }
+    }
+}
+
+impl_plan_tree_node_for_unary! {StreamLocalApproxPercentile}
+
+impl StreamNode for StreamLocalApproxPercentile {
+    fn to_stream_prost_body(&self, _state: &mut BuildFragmentGraphState) -> PbNodeBody {
+        let relative_error = self.relative_error.get_data().as_ref().unwrap();
+        let relative_error = relative_error.as_float64().into_inner();
+        let base = (1.0 + relative_error) / (1.0 - relative_error);
+        let percentile_index = self.percentile_col.index() as u32;
+        let body = LocalApproxPercentileNode {
+            base,
+            percentile_index,
+        };
+        PbNodeBody::LocalApproxPercentile(body)
+    }
+}
+
+impl ExprRewritable for StreamLocalApproxPercentile {
+    fn has_rewritable_expr(&self) -> bool {
+        false
+    }
+
+    fn rewrite_exprs(&self, _rewriter: &mut dyn ExprRewriter) -> PlanRef {
+        unimplemented!()
+    }
+}
+
+impl ExprVisitable for StreamLocalApproxPercentile {
+    fn visit_exprs(&self, _v: &mut dyn ExprVisitor) {}
+}
diff --git a/src/frontend/src/optimizer/plan_node/stream_materialize.rs b/src/frontend/src/optimizer/plan_node/stream_materialize.rs
index e5a2496916adc..865dc71191b46 100644
--- a/src/frontend/src/optimizer/plan_node/stream_materialize.rs
+++ b/src/frontend/src/optimizer/plan_node/stream_materialize.rs
@@ -59,6 +59,7 @@ impl StreamMaterialize {
             input.append_only(),
             input.emit_on_window_close(),
             input.watermark_columns().clone(),
+            input.columns_monotonicity().clone(),
         );
         Self { base, input, table }
     }
diff --git a/src/frontend/src/optimizer/plan_node/stream_now.rs b/src/frontend/src/optimizer/plan_node/stream_now.rs
index 22a0d2c5fb0fb..9ec80d15bac30 100644
--- a/src/frontend/src/optimizer/plan_node/stream_now.rs
+++ b/src/frontend/src/optimizer/plan_node/stream_now.rs
@@ -26,7 +26,7 @@ use super::utils::{childless_record, Distill, TableCatalogBuilder};
 use super::{generic, ExprRewritable, PlanBase, StreamNode};
 use crate::optimizer::plan_node::expr_visitable::ExprVisitable;
 use crate::optimizer::plan_node::utils::column_names_pretty;
-use crate::optimizer::property::Distribution;
+use crate::optimizer::property::{Distribution, Monotonicity, MonotonicityMap};
 use crate::stream_fragmenter::BuildFragmentGraphState;
 
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
@@ -39,12 +39,17 @@ impl StreamNow {
     pub fn new(core: generic::Now) -> Self {
         let mut watermark_columns = FixedBitSet::with_capacity(1);
         watermark_columns.set(0, true);
+
+        let mut columns_monotonicity = MonotonicityMap::new();
+        columns_monotonicity.insert(0, Monotonicity::NonDecreasing);
+
         let base = PlanBase::new_stream_with_core(
             &core,
             Distribution::Single,
             core.mode.is_generate_series(), // append only
             core.mode.is_generate_series(), // emit on window close
             watermark_columns,
+            columns_monotonicity,
         );
         Self { base, core }
     }
diff --git a/src/frontend/src/optimizer/plan_node/stream_over_window.rs b/src/frontend/src/optimizer/plan_node/stream_over_window.rs
index be6c63bcb50de..6b0beaa9f99cc 100644
--- a/src/frontend/src/optimizer/plan_node/stream_over_window.rs
+++ b/src/frontend/src/optimizer/plan_node/stream_over_window.rs
@@ -23,6 +23,7 @@ use super::stream::prelude::*;
 use super::utils::{impl_distill_by_unit, TableCatalogBuilder};
 use super::{generic, ExprRewritable, PlanBase, PlanRef, PlanTreeNodeUnary, StreamNode};
 use crate::optimizer::plan_node::expr_visitable::ExprVisitable;
+use crate::optimizer::property::MonotonicityMap;
 use crate::stream_fragmenter::BuildFragmentGraphState;
 use crate::TableCatalog;
 
@@ -45,6 +46,7 @@ impl StreamOverWindow {
             false, // general over window cannot be append-only
             false,
             watermark_columns,
+            MonotonicityMap::new(), // TODO: derive monotonicity
         );
         StreamOverWindow { base, core }
     }
diff --git a/src/frontend/src/optimizer/plan_node/stream_project.rs b/src/frontend/src/optimizer/plan_node/stream_project.rs
index e8ff1df6e82db..ef879627c66bd 100644
--- a/src/frontend/src/optimizer/plan_node/stream_project.rs
+++ b/src/frontend/src/optimizer/plan_node/stream_project.rs
@@ -20,10 +20,9 @@ use risingwave_pb::stream_plan::ProjectNode;
 use super::stream::prelude::*;
 use super::utils::{childless_record, watermark_pretty, Distill};
 use super::{generic, ExprRewritable, PlanBase, PlanRef, PlanTreeNodeUnary, StreamNode};
-use crate::expr::{
-    try_derive_watermark, Expr, ExprImpl, ExprRewriter, ExprVisitor, WatermarkDerivation,
-};
+use crate::expr::{Expr, ExprImpl, ExprRewriter, ExprVisitor};
 use crate::optimizer::plan_node::expr_visitable::ExprVisitable;
+use crate::optimizer::property::{analyze_monotonicity, monotonicity_variants, MonotonicityMap};
 use crate::stream_fragmenter::BuildFragmentGraphState;
 use crate::utils::ColIndexMappingRewriteExt;
 
@@ -82,23 +81,28 @@ impl StreamProject {
 
         let mut watermark_derivations = vec![];
         let mut nondecreasing_exprs = vec![];
-        let mut watermark_columns = FixedBitSet::with_capacity(core.exprs.len());
+        let mut out_watermark_columns = FixedBitSet::with_capacity(core.exprs.len());
+        let mut out_monotonicity_map = MonotonicityMap::new();
         for (expr_idx, expr) in core.exprs.iter().enumerate() {
-            match try_derive_watermark(expr) {
-                WatermarkDerivation::Watermark(input_idx) => {
-                    if input.watermark_columns().contains(input_idx) {
-                        watermark_derivations.push((input_idx, expr_idx));
-                        watermark_columns.insert(expr_idx);
+            use monotonicity_variants::*;
+            match analyze_monotonicity(expr) {
+                Inherent(monotonicity) => {
+                    out_monotonicity_map.insert(expr_idx, monotonicity);
+                    if monotonicity.is_non_decreasing() && !monotonicity.is_constant() {
+                        // TODO(rc): may be we should also derive watermark for constant later
+                        nondecreasing_exprs.push(expr_idx); // to produce watermarks
+                        out_watermark_columns.insert(expr_idx);
                     }
                 }
-                WatermarkDerivation::Nondecreasing => {
-                    nondecreasing_exprs.push(expr_idx);
-                    watermark_columns.insert(expr_idx);
-                }
-                WatermarkDerivation::Constant => {
-                    // XXX(rc): we can produce one watermark on each recovery for this case.
+                FollowingInput(input_idx) => {
+                    let in_monotonicity = input.columns_monotonicity()[input_idx];
+                    out_monotonicity_map.insert(expr_idx, in_monotonicity);
+                    if input.watermark_columns().contains(input_idx) {
+                        watermark_derivations.push((input_idx, expr_idx)); // to propagate watermarks
+                        out_watermark_columns.insert(expr_idx);
+                    }
                 }
-                WatermarkDerivation::None => {}
+                _FollowingInputInversely(_) => {}
             }
         }
         // Project executor won't change the append-only behavior of the stream, so it depends on
@@ -108,7 +112,8 @@ impl StreamProject {
             distribution,
             input.append_only(),
             input.emit_on_window_close(),
-            watermark_columns,
+            out_watermark_columns,
+            out_monotonicity_map,
         );
 
         StreamProject {
diff --git a/src/frontend/src/optimizer/plan_node/stream_project_set.rs b/src/frontend/src/optimizer/plan_node/stream_project_set.rs
index 2af5d54234363..5735c4b9d5644 100644
--- a/src/frontend/src/optimizer/plan_node/stream_project_set.rs
+++ b/src/frontend/src/optimizer/plan_node/stream_project_set.rs
@@ -20,8 +20,9 @@ use risingwave_pb::stream_plan::ProjectSetNode;
 use super::stream::prelude::*;
 use super::utils::impl_distill_by_unit;
 use super::{generic, ExprRewritable, PlanBase, PlanRef, PlanTreeNodeUnary, StreamNode};
-use crate::expr::{try_derive_watermark, ExprRewriter, ExprVisitor, WatermarkDerivation};
+use crate::expr::{ExprRewriter, ExprVisitor};
 use crate::optimizer::plan_node::expr_visitable::ExprVisitable;
+use crate::optimizer::property::{analyze_monotonicity, monotonicity_variants, MonotonicityMap};
 use crate::stream_fragmenter::BuildFragmentGraphState;
 use crate::utils::ColIndexMappingRewriteExt;
 
@@ -46,23 +47,27 @@ impl StreamProjectSet {
 
         let mut watermark_derivations = vec![];
         let mut nondecreasing_exprs = vec![];
-        let mut watermark_columns = FixedBitSet::with_capacity(core.output_len());
+        let mut out_watermark_columns = FixedBitSet::with_capacity(core.output_len());
         for (expr_idx, expr) in core.select_list.iter().enumerate() {
-            match try_derive_watermark(expr) {
-                WatermarkDerivation::Watermark(input_idx) => {
-                    if input.watermark_columns().contains(input_idx) {
-                        watermark_derivations.push((input_idx, expr_idx));
-                        watermark_columns.insert(expr_idx + 1);
+            let out_expr_idx = expr_idx + 1;
+
+            use monotonicity_variants::*;
+            match analyze_monotonicity(expr) {
+                Inherent(monotonicity) => {
+                    if monotonicity.is_non_decreasing() && !monotonicity.is_constant() {
+                        // TODO(rc): may be we should also derive watermark for constant later
+                        // FIXME(rc): we need to check expr is not table function
+                        nondecreasing_exprs.push(expr_idx); // to produce watermarks
+                        out_watermark_columns.insert(out_expr_idx);
                     }
                 }
-                WatermarkDerivation::Nondecreasing => {
-                    nondecreasing_exprs.push(expr_idx);
-                    watermark_columns.insert(expr_idx + 1);
-                }
-                WatermarkDerivation::Constant => {
-                    // XXX(rc): we can produce one watermark on each recovery for this case.
+                FollowingInput(input_idx) => {
+                    if input.watermark_columns().contains(input_idx) {
+                        watermark_derivations.push((input_idx, expr_idx)); // to propagate watermarks
+                        out_watermark_columns.insert(out_expr_idx);
+                    }
                 }
-                WatermarkDerivation::None => {}
+                _FollowingInputInversely(_) => {}
             }
         }
 
@@ -73,7 +78,8 @@ impl StreamProjectSet {
             distribution,
             input.append_only(),
             input.emit_on_window_close(),
-            watermark_columns,
+            out_watermark_columns,
+            MonotonicityMap::new(), // TODO: derive monotonicity
         );
         StreamProjectSet {
             base,
diff --git a/src/frontend/src/optimizer/plan_node/stream_row_id_gen.rs b/src/frontend/src/optimizer/plan_node/stream_row_id_gen.rs
index bf4bafeed26d0..36b96f4dad36e 100644
--- a/src/frontend/src/optimizer/plan_node/stream_row_id_gen.rs
+++ b/src/frontend/src/optimizer/plan_node/stream_row_id_gen.rs
@@ -50,6 +50,7 @@ impl StreamRowIdGen {
             input.append_only(),
             input.emit_on_window_close(),
             input.watermark_columns().clone(),
+            input.columns_monotonicity().clone(),
         );
         Self {
             base,
diff --git a/src/frontend/src/optimizer/plan_node/stream_row_merge.rs b/src/frontend/src/optimizer/plan_node/stream_row_merge.rs
new file mode 100644
index 0000000000000..f4c2c894fdf66
--- /dev/null
+++ b/src/frontend/src/optimizer/plan_node/stream_row_merge.rs
@@ -0,0 +1,160 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use anyhow::anyhow;
+use fixedbitset::FixedBitSet;
+use pretty_xmlish::{Pretty, XmlNode};
+use risingwave_common::bail;
+use risingwave_common::catalog::Schema;
+use risingwave_common::util::column_index_mapping::ColIndexMapping;
+use risingwave_pb::stream_plan::stream_node::PbNodeBody;
+
+use crate::error::Result;
+use crate::expr::{ExprRewriter, ExprVisitor};
+use crate::optimizer::plan_node::expr_visitable::ExprVisitable;
+use crate::optimizer::plan_node::generic::{GenericPlanRef, PhysicalPlanRef};
+use crate::optimizer::plan_node::stream::StreamPlanRef;
+use crate::optimizer::plan_node::utils::{childless_record, Distill};
+use crate::optimizer::plan_node::{
+    ExprRewritable, PlanBase, PlanTreeNodeBinary, Stream, StreamNode,
+};
+use crate::stream_fragmenter::BuildFragmentGraphState;
+use crate::PlanRef;
+
+/// `StreamRowMerge` is used for merging two streams with the same stream key and distribution.
+/// It will buffer the outputs from its input streams until we receive a barrier.
+/// On receiving a barrier, it will `Project` their outputs according
+/// to the provided `lhs_mapping` and `rhs_mapping`.
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct StreamRowMerge {
+    pub base: PlanBase<Stream>,
+    pub lhs_input: PlanRef,
+    pub rhs_input: PlanRef,
+    /// Maps input from the lhs to the output.
+    pub lhs_mapping: ColIndexMapping,
+    /// Maps input from the rhs to the output.
+    pub rhs_mapping: ColIndexMapping,
+}
+
+impl StreamRowMerge {
+    pub fn new(
+        lhs_input: PlanRef,
+        rhs_input: PlanRef,
+        lhs_mapping: ColIndexMapping,
+        rhs_mapping: ColIndexMapping,
+    ) -> Result<Self> {
+        assert_eq!(lhs_mapping.target_size(), rhs_mapping.target_size());
+        assert_eq!(lhs_input.distribution(), rhs_input.distribution());
+        assert_eq!(lhs_input.stream_key(), rhs_input.stream_key());
+        let mut schema_fields = Vec::with_capacity(lhs_mapping.target_size());
+        let o2i_lhs = lhs_mapping
+            .inverse()
+            .ok_or_else(|| anyhow!("lhs_mapping should be invertible"))?;
+        let o2i_rhs = rhs_mapping
+            .inverse()
+            .ok_or_else(|| anyhow!("rhs_mapping should be invertible"))?;
+        for output_idx in 0..lhs_mapping.target_size() {
+            if let Some(lhs_idx) = o2i_lhs.try_map(output_idx) {
+                schema_fields.push(lhs_input.schema().fields()[lhs_idx].clone());
+            } else if let Some(rhs_idx) = o2i_rhs.try_map(output_idx) {
+                schema_fields.push(rhs_input.schema().fields()[rhs_idx].clone());
+            } else {
+                bail!(
+                    "output index {} not found in either lhs or rhs mapping",
+                    output_idx
+                );
+            }
+        }
+        let schema = Schema::new(schema_fields);
+        assert!(!schema.is_empty());
+        let watermark_columns = FixedBitSet::with_capacity(schema.fields.len());
+
+        let base = PlanBase::new_stream(
+            lhs_input.ctx(),
+            schema,
+            lhs_input.stream_key().map(|k| k.to_vec()),
+            lhs_input.functional_dependency().clone(),
+            lhs_input.distribution().clone(),
+            lhs_input.append_only(),
+            lhs_input.emit_on_window_close(),
+            watermark_columns,
+            lhs_input.columns_monotonicity().clone(),
+        );
+        Ok(Self {
+            base,
+            lhs_input,
+            rhs_input,
+            lhs_mapping,
+            rhs_mapping,
+        })
+    }
+}
+
+impl Distill for StreamRowMerge {
+    fn distill<'a>(&self) -> XmlNode<'a> {
+        let mut out = Vec::with_capacity(1);
+
+        if self.base.ctx().is_explain_verbose() {
+            let f = |t| Pretty::debug(&t);
+            let e = Pretty::Array(self.base.schema().fields().iter().map(f).collect());
+            out = vec![("output", e)];
+        }
+        childless_record("StreamRowMerge", out)
+    }
+}
+
+impl PlanTreeNodeBinary for StreamRowMerge {
+    fn left(&self) -> PlanRef {
+        self.lhs_input.clone()
+    }
+
+    fn right(&self) -> PlanRef {
+        self.rhs_input.clone()
+    }
+
+    fn clone_with_left_right(&self, left: PlanRef, right: PlanRef) -> Self {
+        Self {
+            base: self.base.clone(),
+            lhs_input: left,
+            rhs_input: right,
+            lhs_mapping: self.lhs_mapping.clone(),
+            rhs_mapping: self.rhs_mapping.clone(),
+        }
+    }
+}
+
+impl_plan_tree_node_for_binary! { StreamRowMerge }
+
+impl StreamNode for StreamRowMerge {
+    fn to_stream_prost_body(&self, _state: &mut BuildFragmentGraphState) -> PbNodeBody {
+        PbNodeBody::RowMerge(risingwave_pb::stream_plan::RowMergeNode {
+            lhs_mapping: Some(self.lhs_mapping.to_protobuf()),
+            rhs_mapping: Some(self.rhs_mapping.to_protobuf()),
+        })
+    }
+}
+
+impl ExprRewritable for StreamRowMerge {
+    fn has_rewritable_expr(&self) -> bool {
+        false
+    }
+
+    fn rewrite_exprs(&self, _rewriter: &mut dyn ExprRewriter) -> PlanRef {
+        unimplemented!()
+    }
+}
+
+impl ExprVisitable for StreamRowMerge {
+    fn visit_exprs(&self, _v: &mut dyn ExprVisitor) {}
+}
diff --git a/src/frontend/src/optimizer/plan_node/stream_share.rs b/src/frontend/src/optimizer/plan_node/stream_share.rs
index 5bf575f622bce..7e6f87fa5c271 100644
--- a/src/frontend/src/optimizer/plan_node/stream_share.rs
+++ b/src/frontend/src/optimizer/plan_node/stream_share.rs
@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::cell::RefCell;
+
 use pretty_xmlish::XmlNode;
 use risingwave_pb::stream_plan::stream_node::PbNodeBody;
 use risingwave_pb::stream_plan::PbStreamNode;
@@ -44,11 +46,19 @@ impl StreamShare {
                 input.append_only(),
                 input.emit_on_window_close(),
                 input.watermark_columns().clone(),
+                input.columns_monotonicity().clone(),
             )
         };
 
         StreamShare { base, core }
     }
+
+    pub fn new_from_input(input: PlanRef) -> Self {
+        let core = generic::Share {
+            input: RefCell::new(input),
+        };
+        Self::new(core)
+    }
 }
 
 impl Distill for StreamShare {
diff --git a/src/frontend/src/optimizer/plan_node/stream_simple_agg.rs b/src/frontend/src/optimizer/plan_node/stream_simple_agg.rs
index 28a377ca04cd2..6ecaa4c308f5e 100644
--- a/src/frontend/src/optimizer/plan_node/stream_simple_agg.rs
+++ b/src/frontend/src/optimizer/plan_node/stream_simple_agg.rs
@@ -23,7 +23,7 @@ use super::utils::{childless_record, plan_node_name, Distill};
 use super::{ExprRewritable, PlanBase, PlanRef, PlanTreeNodeUnary, StreamNode};
 use crate::expr::{ExprRewriter, ExprVisitor};
 use crate::optimizer::plan_node::expr_visitable::ExprVisitable;
-use crate::optimizer::property::Distribution;
+use crate::optimizer::property::{Distribution, MonotonicityMap};
 use crate::stream_fragmenter::BuildFragmentGraphState;
 
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
@@ -50,7 +50,14 @@ impl StreamSimpleAgg {
         let watermark_columns = FixedBitSet::with_capacity(core.output_len());
 
         // Simple agg executor might change the append-only behavior of the stream.
-        let base = PlanBase::new_stream_with_core(&core, dist, false, false, watermark_columns);
+        let base = PlanBase::new_stream_with_core(
+            &core,
+            dist,
+            false,
+            false,
+            watermark_columns,
+            MonotonicityMap::new(),
+        );
         StreamSimpleAgg {
             base,
             core,
diff --git a/src/frontend/src/optimizer/plan_node/stream_sink.rs b/src/frontend/src/optimizer/plan_node/stream_sink.rs
index 49378ab0a53ed..dcdd15b067a0e 100644
--- a/src/frontend/src/optimizer/plan_node/stream_sink.rs
+++ b/src/frontend/src/optimizer/plan_node/stream_sink.rs
@@ -49,7 +49,8 @@ use crate::optimizer::plan_node::utils::plan_has_backfill_leaf_nodes;
 use crate::optimizer::plan_node::PlanTreeNodeUnary;
 use crate::optimizer::property::{Distribution, Order, RequiredDist};
 use crate::stream_fragmenter::BuildFragmentGraphState;
-use crate::{TableCatalog, WithOptions};
+use crate::utils::WithOptionsSecResolved;
+use crate::TableCatalog;
 
 const DOWNSTREAM_PK_KEY: &str = "primary_key";
 
@@ -205,7 +206,7 @@ impl StreamSink {
         user_cols: FixedBitSet,
         out_names: Vec<String>,
         definition: String,
-        properties: WithOptions,
+        properties: WithOptionsSecResolved,
         format_desc: Option<SinkFormatDesc>,
         partition_info: Option<PartitionComputeInfo>,
     ) -> Result<Self> {
@@ -308,7 +309,7 @@ impl StreamSink {
         user_order_by: Order,
         columns: Vec<ColumnCatalog>,
         definition: String,
-        properties: WithOptions,
+        properties: WithOptionsSecResolved,
         format_desc: Option<SinkFormatDesc>,
         partition_info: Option<PartitionComputeInfo>,
     ) -> Result<(PlanRef, SinkDesc)> {
@@ -381,6 +382,7 @@ impl StreamSink {
         } else {
             CreateType::Foreground
         };
+        let (properties, secret_refs) = properties.into_parts();
         let sink_desc = SinkDesc {
             id: SinkId::placeholder(),
             name,
@@ -391,7 +393,8 @@ impl StreamSink {
             plan_pk: pk,
             downstream_pk,
             distribution_key,
-            properties: properties.into_inner(),
+            properties,
+            secret_refs,
             sink_type,
             format_desc,
             target_table,
@@ -401,7 +404,7 @@ impl StreamSink {
         Ok((input, sink_desc))
     }
 
-    fn is_user_defined_append_only(properties: &WithOptions) -> Result<bool> {
+    fn is_user_defined_append_only(properties: &WithOptionsSecResolved) -> Result<bool> {
         if let Some(sink_type) = properties.get(SINK_TYPE_OPTION) {
             if sink_type != SINK_TYPE_APPEND_ONLY
                 && sink_type != SINK_TYPE_DEBEZIUM
@@ -423,7 +426,7 @@ impl StreamSink {
         Ok(properties.value_eq_ignore_case(SINK_TYPE_OPTION, SINK_TYPE_APPEND_ONLY))
     }
 
-    fn is_user_force_append_only(properties: &WithOptions) -> Result<bool> {
+    fn is_user_force_append_only(properties: &WithOptionsSecResolved) -> Result<bool> {
         if properties.contains_key(SINK_USER_FORCE_APPEND_ONLY_OPTION)
             && !properties.value_eq_ignore_case(SINK_USER_FORCE_APPEND_ONLY_OPTION, "true")
             && !properties.value_eq_ignore_case(SINK_USER_FORCE_APPEND_ONLY_OPTION, "false")
@@ -442,14 +445,16 @@ impl StreamSink {
 
     fn derive_sink_type(
         input_append_only: bool,
-        properties: &WithOptions,
+        properties: &WithOptionsSecResolved,
         format_desc: Option<&SinkFormatDesc>,
     ) -> Result<SinkType> {
         let frontend_derived_append_only = input_append_only;
         let (user_defined_append_only, user_force_append_only, syntax_legacy) = match format_desc {
             Some(f) => (
                 f.format == SinkFormat::AppendOnly,
-                Self::is_user_force_append_only(&WithOptions::new(f.options.clone()))?,
+                Self::is_user_force_append_only(&WithOptionsSecResolved::without_secrets(
+                    f.options.clone(),
+                ))?,
                 false,
             ),
             None => (
diff --git a/src/frontend/src/optimizer/plan_node/stream_sort.rs b/src/frontend/src/optimizer/plan_node/stream_sort.rs
index 6b45f8fd35a6a..c4acd275f1236 100644
--- a/src/frontend/src/optimizer/plan_node/stream_sort.rs
+++ b/src/frontend/src/optimizer/plan_node/stream_sort.rs
@@ -24,6 +24,7 @@ use super::stream::prelude::*;
 use super::utils::{childless_record, Distill, TableCatalogBuilder};
 use super::{ExprRewritable, PlanBase, PlanRef, PlanTreeNodeUnary, StreamNode};
 use crate::optimizer::plan_node::expr_visitable::ExprVisitable;
+use crate::optimizer::property::{Monotonicity, MonotonicityMap};
 use crate::stream_fragmenter::BuildFragmentGraphState;
 use crate::TableCatalog;
 
@@ -53,8 +54,14 @@ impl StreamEowcSort {
         let stream_key = input.stream_key().map(|v| v.to_vec());
         let fd_set = input.functional_dependency().clone();
         let dist = input.distribution().clone();
+
         let mut watermark_columns = FixedBitSet::with_capacity(input.schema().len());
         watermark_columns.insert(sort_column_index);
+
+        // StreamEowcSort makes the sorting watermark column non-decreasing
+        let mut columns_monotonicity = MonotonicityMap::new();
+        columns_monotonicity.insert(sort_column_index, Monotonicity::NonDecreasing);
+
         let base = PlanBase::new_stream(
             input.ctx(),
             schema,
@@ -64,6 +71,7 @@ impl StreamEowcSort {
             true,
             true,
             watermark_columns,
+            columns_monotonicity,
         );
         Self {
             base,
diff --git a/src/frontend/src/optimizer/plan_node/stream_source.rs b/src/frontend/src/optimizer/plan_node/stream_source.rs
index 83ad14886872a..d7808e4be51ce 100644
--- a/src/frontend/src/optimizer/plan_node/stream_source.rs
+++ b/src/frontend/src/optimizer/plan_node/stream_source.rs
@@ -29,7 +29,7 @@ use super::{generic, ExprRewritable, PlanBase, StreamNode};
 use crate::catalog::source_catalog::SourceCatalog;
 use crate::optimizer::plan_node::expr_visitable::ExprVisitable;
 use crate::optimizer::plan_node::utils::column_names_pretty;
-use crate::optimizer::property::Distribution;
+use crate::optimizer::property::{Distribution, MonotonicityMap};
 use crate::stream_fragmenter::BuildFragmentGraphState;
 
 /// [`StreamSource`] represents a table/connector source at the very beginning of the graph.
@@ -64,6 +64,7 @@ impl StreamSource {
             core.catalog.as_ref().map_or(true, |s| s.append_only),
             false,
             FixedBitSet::with_capacity(core.column_catalog.len()),
+            MonotonicityMap::new(),
         );
         Self { base, core }
     }
@@ -91,25 +92,29 @@ impl Distill for StreamSource {
 impl StreamNode for StreamSource {
     fn to_stream_prost_body(&self, state: &mut BuildFragmentGraphState) -> PbNodeBody {
         let source_catalog = self.source_catalog();
-        let source_inner = source_catalog.map(|source_catalog| PbStreamSource {
-            source_id: source_catalog.id,
-            source_name: source_catalog.name.clone(),
-            state_table: Some(
-                generic::Source::infer_internal_table_catalog(false)
-                    .with_id(state.gen_table_id_wrapped())
-                    .to_internal_table_prost(),
-            ),
-            info: Some(source_catalog.info.clone()),
-            row_id_index: self.core.row_id_index.map(|index| index as _),
-            columns: self
-                .core
-                .column_catalog
-                .iter()
-                .map(|c| c.to_protobuf())
-                .collect_vec(),
-            with_properties: source_catalog.with_properties.clone().into_iter().collect(),
-            rate_limit: self.base.ctx().overwrite_options().streaming_rate_limit,
-            secret_refs: Default::default(),
+        let source_inner = source_catalog.map(|source_catalog| {
+            let (with_properties, secret_refs) =
+                source_catalog.with_properties.clone().into_parts();
+            PbStreamSource {
+                source_id: source_catalog.id,
+                source_name: source_catalog.name.clone(),
+                state_table: Some(
+                    generic::Source::infer_internal_table_catalog(false)
+                        .with_id(state.gen_table_id_wrapped())
+                        .to_internal_table_prost(),
+                ),
+                info: Some(source_catalog.info.clone()),
+                row_id_index: self.core.row_id_index.map(|index| index as _),
+                columns: self
+                    .core
+                    .column_catalog
+                    .iter()
+                    .map(|c| c.to_protobuf())
+                    .collect_vec(),
+                with_properties,
+                rate_limit: source_catalog.rate_limit,
+                secret_refs,
+            }
         });
         PbNodeBody::Source(SourceNode { source_inner })
     }
diff --git a/src/frontend/src/optimizer/plan_node/stream_source_scan.rs b/src/frontend/src/optimizer/plan_node/stream_source_scan.rs
index 0449648798ea7..83c79259952b2 100644
--- a/src/frontend/src/optimizer/plan_node/stream_source_scan.rs
+++ b/src/frontend/src/optimizer/plan_node/stream_source_scan.rs
@@ -32,7 +32,7 @@ use crate::catalog::source_catalog::SourceCatalog;
 use crate::optimizer::plan_node::expr_visitable::ExprVisitable;
 use crate::optimizer::plan_node::utils::{childless_record, Distill};
 use crate::optimizer::plan_node::{generic, ExprRewritable, StreamNode};
-use crate::optimizer::property::Distribution;
+use crate::optimizer::property::{Distribution, MonotonicityMap};
 use crate::scheduler::SchedulerResult;
 use crate::stream_fragmenter::BuildFragmentGraphState;
 use crate::{Explain, TableCatalog};
@@ -75,6 +75,7 @@ impl StreamSourceScan {
             core.catalog.as_ref().map_or(true, |s| s.append_only),
             false,
             FixedBitSet::with_capacity(core.column_catalog.len()),
+            MonotonicityMap::new(),
         );
 
         Self { base, core }
@@ -139,6 +140,7 @@ impl StreamSourceScan {
             .collect_vec();
 
         let source_catalog = self.source_catalog();
+        let (with_properties, secret_refs) = source_catalog.with_properties.clone().into_parts();
         let backfill = SourceBackfillNode {
             upstream_source_id: source_catalog.id,
             source_name: source_catalog.name.clone(),
@@ -155,9 +157,9 @@ impl StreamSourceScan {
                 .iter()
                 .map(|c| c.to_protobuf())
                 .collect_vec(),
-            with_properties: source_catalog.with_properties.clone().into_iter().collect(),
-            rate_limit: self.base.ctx().overwrite_options().streaming_rate_limit,
-            secret_refs: Default::default(),
+            with_properties,
+            rate_limit: self.base.ctx().overwrite_options().backfill_rate_limit,
+            secret_refs,
         };
 
         let fields = self.schema().to_prost();
diff --git a/src/frontend/src/optimizer/plan_node/stream_stateless_simple_agg.rs b/src/frontend/src/optimizer/plan_node/stream_stateless_simple_agg.rs
index 8ce0997b7fe1f..93c56efad3d5f 100644
--- a/src/frontend/src/optimizer/plan_node/stream_stateless_simple_agg.rs
+++ b/src/frontend/src/optimizer/plan_node/stream_stateless_simple_agg.rs
@@ -22,7 +22,7 @@ use super::utils::impl_distill_by_unit;
 use super::{ExprRewritable, PlanBase, PlanRef, PlanTreeNodeUnary, StreamNode};
 use crate::expr::{ExprRewriter, ExprVisitor};
 use crate::optimizer::plan_node::expr_visitable::ExprVisitable;
-use crate::optimizer::property::RequiredDist;
+use crate::optimizer::property::{MonotonicityMap, RequiredDist};
 use crate::stream_fragmenter::BuildFragmentGraphState;
 
 /// Streaming stateless simple agg.
@@ -57,6 +57,7 @@ impl StreamStatelessSimpleAgg {
             input.append_only(),
             input.emit_on_window_close(),
             watermark_columns,
+            MonotonicityMap::new(),
         );
         StreamStatelessSimpleAgg { base, core }
     }
diff --git a/src/frontend/src/optimizer/plan_node/stream_table_scan.rs b/src/frontend/src/optimizer/plan_node/stream_table_scan.rs
index 1e93514f6c0f7..8a85a0c324dca 100644
--- a/src/frontend/src/optimizer/plan_node/stream_table_scan.rs
+++ b/src/frontend/src/optimizer/plan_node/stream_table_scan.rs
@@ -31,7 +31,7 @@ use crate::catalog::ColumnId;
 use crate::expr::{ExprRewriter, ExprVisitor, FunctionCall};
 use crate::optimizer::plan_node::expr_visitable::ExprVisitable;
 use crate::optimizer::plan_node::utils::{IndicesDisplay, TableCatalogBuilder};
-use crate::optimizer::property::{Distribution, DistributionDisplay};
+use crate::optimizer::property::{Distribution, DistributionDisplay, MonotonicityMap};
 use crate::scheduler::SchedulerResult;
 use crate::stream_fragmenter::BuildFragmentGraphState;
 use crate::TableCatalog;
@@ -74,6 +74,7 @@ impl StreamTableScan {
             core.append_only(),
             false,
             core.watermark_columns(),
+            MonotonicityMap::new(),
         );
         Self {
             base,
@@ -319,7 +320,7 @@ impl StreamTableScan {
             table_desc: Some(self.core.table_desc.try_to_protobuf()?),
             state_table: Some(catalog),
             arrangement_table,
-            rate_limit: self.base.ctx().overwrite_options().streaming_rate_limit,
+            rate_limit: self.base.ctx().overwrite_options().backfill_rate_limit,
             ..Default::default()
         });
 
diff --git a/src/frontend/src/optimizer/plan_node/stream_temporal_join.rs b/src/frontend/src/optimizer/plan_node/stream_temporal_join.rs
index f94dbba36cb79..7f50d4b27e625 100644
--- a/src/frontend/src/optimizer/plan_node/stream_temporal_join.rs
+++ b/src/frontend/src/optimizer/plan_node/stream_temporal_join.rs
@@ -69,12 +69,19 @@ impl StreamTemporalJoin {
                 .rewrite_bitset(core.left.watermark_columns()),
         );
 
+        let columns_monotonicity = core.i2o_col_mapping().rewrite_monotonicity_map(
+            &core
+                .l2i_col_mapping()
+                .rewrite_monotonicity_map(core.left.columns_monotonicity()),
+        );
+
         let base = PlanBase::new_stream_with_core(
             &core,
             dist,
             append_only,
             false, // TODO(rc): derive EOWC property from input
             watermark_columns,
+            columns_monotonicity,
         );
 
         Self {
diff --git a/src/frontend/src/optimizer/plan_node/stream_topn.rs b/src/frontend/src/optimizer/plan_node/stream_topn.rs
index 9581c07ff297b..80ca9141033c5 100644
--- a/src/frontend/src/optimizer/plan_node/stream_topn.rs
+++ b/src/frontend/src/optimizer/plan_node/stream_topn.rs
@@ -21,7 +21,7 @@ use super::stream::prelude::*;
 use super::utils::{plan_node_name, Distill};
 use super::{generic, ExprRewritable, PlanBase, PlanRef, PlanTreeNodeUnary, StreamNode};
 use crate::optimizer::plan_node::expr_visitable::ExprVisitable;
-use crate::optimizer::property::{Distribution, Order};
+use crate::optimizer::property::{Distribution, MonotonicityMap, Order};
 use crate::stream_fragmenter::BuildFragmentGraphState;
 
 /// `StreamTopN` implements [`super::LogicalTopN`] to find the top N elements with a heap
@@ -42,7 +42,14 @@ impl StreamTopN {
         };
         let watermark_columns = FixedBitSet::with_capacity(input.schema().len());
 
-        let base = PlanBase::new_stream_with_core(&core, dist, false, false, watermark_columns);
+        let base = PlanBase::new_stream_with_core(
+            &core,
+            dist,
+            false,
+            false,
+            watermark_columns,
+            MonotonicityMap::new(),
+        );
         StreamTopN { base, core }
     }
 
diff --git a/src/frontend/src/optimizer/plan_node/stream_union.rs b/src/frontend/src/optimizer/plan_node/stream_union.rs
index 1c269ec0c5ad2..2e424fc0604b6 100644
--- a/src/frontend/src/optimizer/plan_node/stream_union.rs
+++ b/src/frontend/src/optimizer/plan_node/stream_union.rs
@@ -25,7 +25,7 @@ use super::{generic, ExprRewritable, PlanRef};
 use crate::optimizer::plan_node::expr_visitable::ExprVisitable;
 use crate::optimizer::plan_node::generic::GenericPlanNode;
 use crate::optimizer::plan_node::{PlanBase, PlanTreeNode, StreamNode};
-use crate::optimizer::property::Distribution;
+use crate::optimizer::property::{Distribution, MonotonicityMap};
 use crate::stream_fragmenter::BuildFragmentGraphState;
 
 /// `StreamUnion` implements [`super::LogicalUnion`]
@@ -61,6 +61,7 @@ impl StreamUnion {
             inputs.iter().all(|x| x.append_only()),
             inputs.iter().all(|x| x.emit_on_window_close()),
             watermark_columns,
+            MonotonicityMap::new(),
         );
 
         StreamUnion { base, core }
diff --git a/src/frontend/src/optimizer/plan_node/stream_values.rs b/src/frontend/src/optimizer/plan_node/stream_values.rs
index 05cb1659b96ef..0a71c208c32ee 100644
--- a/src/frontend/src/optimizer/plan_node/stream_values.rs
+++ b/src/frontend/src/optimizer/plan_node/stream_values.rs
@@ -23,7 +23,7 @@ use super::utils::{childless_record, Distill};
 use super::{ExprRewritable, LogicalValues, PlanBase, StreamNode};
 use crate::expr::{Expr, ExprImpl, ExprVisitor};
 use crate::optimizer::plan_node::expr_visitable::ExprVisitable;
-use crate::optimizer::property::Distribution;
+use crate::optimizer::property::{Distribution, MonotonicityMap};
 use crate::stream_fragmenter::BuildFragmentGraphState;
 
 /// `StreamValues` implements `LogicalValues.to_stream()`
@@ -48,6 +48,7 @@ impl StreamValues {
             true,
             false,
             FixedBitSet::with_capacity(logical.schema().len()),
+            MonotonicityMap::new(),
         );
         Self { base, logical }
     }
diff --git a/src/frontend/src/optimizer/plan_node/stream_watermark_filter.rs b/src/frontend/src/optimizer/plan_node/stream_watermark_filter.rs
index ffb08776b3fe5..7ea0dbaf6dd95 100644
--- a/src/frontend/src/optimizer/plan_node/stream_watermark_filter.rs
+++ b/src/frontend/src/optimizer/plan_node/stream_watermark_filter.rs
@@ -49,6 +49,8 @@ impl StreamWatermarkFilter {
             input.append_only(),
             false, // TODO(rc): decide EOWC property
             watermark_columns,
+            // watermark filter preserves input order and hence monotonicity
+            input.columns_monotonicity().clone(),
         );
         Self::with_base(base, input, watermark_descs)
     }
diff --git a/src/frontend/src/optimizer/plan_node/utils.rs b/src/frontend/src/optimizer/plan_node/utils.rs
index 9ac61ed759c6d..2ac317d597bad 100644
--- a/src/frontend/src/optimizer/plan_node/utils.rs
+++ b/src/frontend/src/optimizer/plan_node/utils.rs
@@ -16,6 +16,7 @@ use std::collections::HashMap;
 use std::default::Default;
 use std::vec;
 
+use anyhow::anyhow;
 use fixedbitset::FixedBitSet;
 use itertools::Itertools;
 use pretty_xmlish::{Pretty, Str, StrAssocArr, XmlNode};
@@ -105,11 +106,6 @@ impl TableCatalogBuilder {
         self.value_indices = Some(value_indices);
     }
 
-    #[allow(dead_code)]
-    pub fn set_watermark_columns(&mut self, watermark_columns: FixedBitSet) {
-        self.watermark_columns = Some(watermark_columns);
-    }
-
     pub fn set_dist_key_in_pk(&mut self, dist_key_in_pk: Vec<usize>) {
         self.dist_key_in_pk = Some(dist_key_in_pk);
     }
@@ -235,21 +231,22 @@ pub(crate) fn watermark_pretty<'a>(
     watermark_columns: &FixedBitSet,
     schema: &Schema,
 ) -> Option<Pretty<'a>> {
-    if watermark_columns.count_ones(..) > 0 {
-        Some(watermark_fields_pretty(watermark_columns.ones(), schema))
-    } else {
-        None
-    }
+    iter_fields_pretty(watermark_columns.ones(), schema)
 }
-pub(crate) fn watermark_fields_pretty<'a>(
-    watermark_columns: impl Iterator<Item = usize>,
+
+pub(crate) fn iter_fields_pretty<'a>(
+    columns: impl Iterator<Item = usize>,
     schema: &Schema,
-) -> Pretty<'a> {
-    let arr = watermark_columns
+) -> Option<Pretty<'a>> {
+    let arr = columns
         .map(|idx| FieldDisplay(schema.fields.get(idx).unwrap()))
         .map(|d| Pretty::display(&d))
-        .collect();
-    Pretty::Array(arr)
+        .collect::<Vec<_>>();
+    if arr.is_empty() {
+        None
+    } else {
+        Some(Pretty::Array(arr))
+    }
 }
 
 #[derive(Clone, Copy)]
@@ -290,14 +287,13 @@ pub(crate) fn sum_affected_row(dml: PlanRef) -> Result<PlanRef> {
     let dml = RequiredDist::single().enforce_if_not_satisfies(dml, &Order::any())?;
     // Accumulate the affected rows.
     let sum_agg = PlanAggCall {
-        agg_kind: AggKind::Sum,
+        agg_kind: PbAggKind::Sum.into(),
         return_type: DataType::Int64,
         inputs: vec![InputRef::new(0, DataType::Int64)],
         distinct: false,
         order_by: vec![],
         filter: Condition::true_cond(),
         direct_args: vec![],
-        user_defined: None,
     };
     let agg = Agg::new(vec![sum_agg], IndexSet::empty(), dml);
     let batch_agg = BatchSimpleAgg::new(agg);
@@ -324,12 +320,16 @@ macro_rules! plan_node_name {
     };
 }
 pub(crate) use plan_node_name;
-use risingwave_common::types::DataType;
-use risingwave_expr::aggregate::AggKind;
+use risingwave_common::license::Feature;
+use risingwave_common::types::{DataType, Interval};
+use risingwave_expr::aggregate::PbAggKind;
+use risingwave_pb::plan_common::as_of::AsOfType;
+use risingwave_pb::plan_common::{as_of, PbAsOf};
+use risingwave_sqlparser::ast::AsOf;
 
 use super::generic::{self, GenericPlanRef, PhysicalPlanRef};
 use super::pretty_config;
-use crate::error::Result;
+use crate::error::{ErrorCode, Result};
 use crate::expr::InputRef;
 use crate::optimizer::plan_node::generic::Agg;
 use crate::optimizer::plan_node::{BatchSimpleAgg, PlanAggCall};
@@ -388,3 +388,52 @@ pub(crate) fn plan_has_backfill_leaf_nodes(plan: &PlanRef) -> bool {
         plan.inputs().iter().all(plan_has_backfill_leaf_nodes)
     }
 }
+
+pub fn to_pb_time_travel_as_of(a: &Option<AsOf>) -> Result<Option<PbAsOf>> {
+    let Some(ref a) = a else {
+        return Ok(None);
+    };
+    Feature::TimeTravel
+        .check_available()
+        .map_err(|e| anyhow::anyhow!(e))?;
+    let as_of_type = match a {
+        AsOf::ProcessTime => {
+            return Err(ErrorCode::NotSupported(
+                "do not support as of proctime".to_string(),
+                "please use as of timestamp".to_string(),
+            )
+            .into());
+        }
+        AsOf::TimestampNum(ts) => AsOfType::Timestamp(as_of::Timestamp { timestamp: *ts }),
+        AsOf::TimestampString(ts) => {
+            let date_time = speedate::DateTime::parse_str_rfc3339(ts)
+                .map_err(|_e| anyhow!("fail to parse timestamp"))?;
+            AsOfType::Timestamp(as_of::Timestamp {
+                timestamp: date_time.timestamp_tz(),
+            })
+        }
+        AsOf::VersionNum(_) | AsOf::VersionString(_) => {
+            return Err(ErrorCode::NotSupported(
+                "do not support as of version".to_string(),
+                "please use as of timestamp".to_string(),
+            )
+            .into());
+        }
+        AsOf::ProcessTimeWithInterval((value, leading_field)) => {
+            let interval = Interval::parse_with_fields(
+                value,
+                Some(crate::Binder::bind_date_time_field(leading_field.clone())),
+            )
+            .map_err(|_| anyhow!("fail to parse interval"))?;
+            let interval_sec = (interval.epoch_in_micros() / 1_000_000) as i64;
+            let timestamp = chrono::Utc::now()
+                .timestamp()
+                .checked_sub(interval_sec)
+                .ok_or_else(|| anyhow!("invalid timestamp"))?;
+            AsOfType::Timestamp(as_of::Timestamp { timestamp })
+        }
+    };
+    Ok(Some(PbAsOf {
+        as_of_type: Some(as_of_type),
+    }))
+}
diff --git a/src/frontend/src/optimizer/plan_visitor/distributed_dml_visitor.rs b/src/frontend/src/optimizer/plan_visitor/distributed_dml_visitor.rs
index f79bf59789969..672e886862d84 100644
--- a/src/frontend/src/optimizer/plan_visitor/distributed_dml_visitor.rs
+++ b/src/frontend/src/optimizer/plan_visitor/distributed_dml_visitor.rs
@@ -40,10 +40,7 @@ impl DistributedDmlVisitor {
     }
 
     fn is_iceberg_source(source_catalog: &Rc<SourceCatalog>) -> bool {
-        let property = ConnectorProperties::extract(
-            source_catalog.with_properties.clone().into_iter().collect(),
-            false,
-        );
+        let property = ConnectorProperties::extract(source_catalog.with_properties.clone(), false);
         if let Ok(property) = property {
             matches!(property, ConnectorProperties::Iceberg(_))
         } else {
diff --git a/src/frontend/src/optimizer/property/mod.rs b/src/frontend/src/optimizer/property/mod.rs
index 69871d0003596..ae6ebe7e8288a 100644
--- a/src/frontend/src/optimizer/property/mod.rs
+++ b/src/frontend/src/optimizer/property/mod.rs
@@ -32,3 +32,5 @@ mod func_dep;
 pub use func_dep::*;
 mod cardinality;
 pub use cardinality::*;
+mod monotonicity;
+pub use monotonicity::*;
diff --git a/src/frontend/src/optimizer/property/monotonicity.rs b/src/frontend/src/optimizer/property/monotonicity.rs
new file mode 100644
index 0000000000000..d3091ca029db3
--- /dev/null
+++ b/src/frontend/src/optimizer/property/monotonicity.rs
@@ -0,0 +1,360 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::BTreeMap;
+use std::ops::Index;
+
+use enum_as_inner::EnumAsInner;
+use risingwave_common::types::DataType;
+use risingwave_pb::expr::expr_node::Type as ExprType;
+
+use crate::expr::{Expr, ExprImpl, FunctionCall, TableFunction};
+
+/// Represents the derivation of the monotonicity of a column.
+/// This enum aims to unify the "non-decreasing analysis" and watermark derivation.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, EnumAsInner)]
+pub enum MonotonicityDerivation {
+    /// The monotonicity of the column is inherent, meaning that it is derived from the column itself.
+    Inherent(Monotonicity),
+    /// The monotonicity of the column follows the monotonicity of the specified column in the input.
+    FollowingInput(usize),
+    /// The monotonicity of the column INVERSELY follows the monotonicity of the specified column in the input.
+    /// This is not used currently.
+    _FollowingInputInversely(usize),
+}
+
+impl MonotonicityDerivation {
+    pub fn inverse(self) -> Self {
+        use MonotonicityDerivation::*;
+        match self {
+            Inherent(monotonicity) => Inherent(monotonicity.inverse()),
+            FollowingInput(idx) => _FollowingInputInversely(idx),
+            _FollowingInputInversely(idx) => FollowingInput(idx),
+        }
+    }
+}
+
+/// Represents the monotonicity of a column.
+///
+/// Monotonicity is a property of the output column of stream node that describes the the order
+/// of the values in the column. One [`Monotonicity`] value is associated with one column, so
+/// each stream node should have a [`MonotonicityMap`] to describe the monotonicity of all its
+/// output columns.
+///
+/// For operator that yields append-only stream, the monotonicity being `NonDecreasing` means
+/// that it will never yield a row smaller than any previously yielded row.
+///
+/// For operator that yields non-append-only stream, the monotonicity being `NonDecreasing` means
+/// that it will never yield a change that has smaller value than any previously yielded change,
+/// ignoring the `Op`. So if such operator yields a `NonDecreasing` column, `Delete` and `UpdateDelete`s
+/// can only happen on the last emitted row (or last rows with the same value on the column). This
+/// is especially useful for `StreamNow` operator with `UpdateCurrent` mode, in which case only
+/// one output row is actively maintained and the value is non-decreasing.
+///
+/// Monotonicity property is be considered in default order type, i.e., ASC NULLS LAST. This means
+/// that `NULL`s are considered largest when analyzing monotonicity.
+///
+/// For distributed operators, the monotonicity describes the property of the output column of
+/// each shard of the operator.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub enum Monotonicity {
+    Constant,
+    NonDecreasing,
+    NonIncreasing,
+    Unknown,
+}
+
+impl Monotonicity {
+    pub fn is_constant(self) -> bool {
+        matches!(self, Monotonicity::Constant)
+    }
+
+    pub fn is_non_decreasing(self) -> bool {
+        // we don't use `EnumAsInner` here because we need to include `Constant`
+        matches!(self, Monotonicity::NonDecreasing | Monotonicity::Constant)
+    }
+
+    pub fn is_non_increasing(self) -> bool {
+        // similar to `is_non_decreasing`
+        matches!(self, Monotonicity::NonIncreasing | Monotonicity::Constant)
+    }
+
+    pub fn is_unknown(self) -> bool {
+        matches!(self, Monotonicity::Unknown)
+    }
+
+    pub fn inverse(self) -> Self {
+        use Monotonicity::*;
+        match self {
+            Constant => Constant,
+            NonDecreasing => NonIncreasing,
+            NonIncreasing => NonDecreasing,
+            Unknown => Unknown,
+        }
+    }
+}
+
+pub mod monotonicity_variants {
+    pub use super::Monotonicity::*;
+    pub use super::MonotonicityDerivation::*;
+}
+
+/// Analyze the monotonicity of an expression.
+pub fn analyze_monotonicity(expr: &ExprImpl) -> MonotonicityDerivation {
+    let analyzer = MonotonicityAnalyzer {};
+    analyzer.visit_expr(expr)
+}
+
+struct MonotonicityAnalyzer {}
+
+impl MonotonicityAnalyzer {
+    fn visit_expr(&self, expr: &ExprImpl) -> MonotonicityDerivation {
+        use monotonicity_variants::*;
+        match expr {
+            // recursion base
+            ExprImpl::InputRef(inner) => FollowingInput(inner.index()),
+            ExprImpl::Literal(_) => Inherent(Constant),
+            ExprImpl::Now(_) => Inherent(NonDecreasing),
+            ExprImpl::UserDefinedFunction(_) => Inherent(Unknown),
+
+            // recursively visit children
+            ExprImpl::FunctionCall(inner) => self.visit_function_call(inner),
+            ExprImpl::FunctionCallWithLambda(inner) => self.visit_function_call(inner.base()),
+            ExprImpl::TableFunction(inner) => self.visit_table_function(inner),
+
+            // the analyzer is not expected to be used when the following expression types are present
+            ExprImpl::Subquery(_)
+            | ExprImpl::AggCall(_)
+            | ExprImpl::CorrelatedInputRef(_)
+            | ExprImpl::WindowFunction(_)
+            | ExprImpl::Parameter(_) => panic!(
+                "Expression `{}` is not expected in the monotonicity analyzer",
+                expr.variant_name()
+            ),
+        }
+    }
+
+    fn visit_unary_op(&self, inputs: &[ExprImpl]) -> MonotonicityDerivation {
+        assert_eq!(inputs.len(), 1);
+        self.visit_expr(&inputs[0])
+    }
+
+    fn visit_binary_op(
+        &self,
+        inputs: &[ExprImpl],
+    ) -> (MonotonicityDerivation, MonotonicityDerivation) {
+        assert_eq!(inputs.len(), 2);
+        (self.visit_expr(&inputs[0]), self.visit_expr(&inputs[1]))
+    }
+
+    fn visit_ternary_op(
+        &self,
+        inputs: &[ExprImpl],
+    ) -> (
+        MonotonicityDerivation,
+        MonotonicityDerivation,
+        MonotonicityDerivation,
+    ) {
+        assert_eq!(inputs.len(), 3);
+        (
+            self.visit_expr(&inputs[0]),
+            self.visit_expr(&inputs[1]),
+            self.visit_expr(&inputs[2]),
+        )
+    }
+
+    fn visit_function_call(&self, func_call: &FunctionCall) -> MonotonicityDerivation {
+        use monotonicity_variants::*;
+
+        fn time_zone_is_without_dst(time_zone: Option<&str>) -> bool {
+            #[allow(clippy::let_and_return)] // to make it more readable
+            let tz_is_utc = time_zone.map_or(
+                false, // conservative
+                |time_zone| time_zone.eq_ignore_ascii_case("UTC"),
+            );
+            tz_is_utc // conservative
+        }
+
+        match func_call.func_type() {
+            ExprType::Unspecified => unreachable!(),
+            ExprType::Add => match self.visit_binary_op(func_call.inputs()) {
+                (Inherent(Constant), any) | (any, Inherent(Constant)) => any,
+                (Inherent(NonDecreasing), Inherent(NonDecreasing)) => Inherent(NonDecreasing),
+                (Inherent(NonIncreasing), Inherent(NonIncreasing)) => Inherent(NonIncreasing),
+                _ => Inherent(Unknown),
+            },
+            ExprType::Subtract => match self.visit_binary_op(func_call.inputs()) {
+                (any, Inherent(Constant)) => any,
+                (Inherent(Constant), any) => any.inverse(),
+                _ => Inherent(Unknown),
+            },
+            ExprType::Multiply | ExprType::Divide | ExprType::Modulus => {
+                match self.visit_binary_op(func_call.inputs()) {
+                    (Inherent(Constant), Inherent(Constant)) => Inherent(Constant),
+                    _ => Inherent(Unknown), // let's be lazy here
+                }
+            }
+            ExprType::TumbleStart => {
+                if func_call.inputs().len() == 2 {
+                    // without `offset`, args: `(start, interval)`
+                    match self.visit_binary_op(func_call.inputs()) {
+                        (any, Inherent(Constant)) => any,
+                        _ => Inherent(Unknown),
+                    }
+                } else {
+                    // with `offset`, args: `(start, interval, offset)`
+                    assert_eq!(ExprType::TumbleStart, func_call.func_type());
+                    match self.visit_ternary_op(func_call.inputs()) {
+                        (any, Inherent(Constant), Inherent(Constant)) => any,
+                        _ => Inherent(Unknown),
+                    }
+                }
+            }
+            ExprType::AtTimeZone => match self.visit_binary_op(func_call.inputs()) {
+                (Inherent(Constant), Inherent(Constant)) => Inherent(Constant),
+                (any, Inherent(Constant)) => {
+                    let time_zone = func_call.inputs()[1]
+                        .as_literal()
+                        .and_then(|literal| literal.get_data().as_ref())
+                        .map(|tz| tz.as_utf8().as_ref());
+                    // 1. For at_time_zone(timestamp, const timezone) -> timestamptz, when timestamp has some monotonicity,
+                    // the result should have the same monotonicity.
+                    // 2. For at_time_zone(timestamptz, const timezone) -> timestamp, when timestamptz has some monotonicity,
+                    // the result only have the same monotonicity when the timezone is without DST (Daylight Saving Time).
+                    if (func_call.inputs()[0].return_type() == DataType::Timestamp
+                        && func_call.return_type() == DataType::Timestamptz)
+                        || time_zone_is_without_dst(time_zone)
+                    {
+                        any
+                    } else {
+                        Inherent(Unknown)
+                    }
+                }
+                _ => Inherent(Unknown),
+            },
+            ExprType::DateTrunc => match func_call.inputs().len() {
+                2 => match self.visit_binary_op(func_call.inputs()) {
+                    (Inherent(Constant), any) => any,
+                    _ => Inherent(Unknown),
+                },
+                3 => match self.visit_ternary_op(func_call.inputs()) {
+                    (Inherent(Constant), Inherent(Constant), Inherent(Constant)) => {
+                        Inherent(Constant)
+                    }
+                    (Inherent(Constant), any, Inherent(Constant)) => {
+                        let time_zone = func_call.inputs()[2]
+                            .as_literal()
+                            .and_then(|literal| literal.get_data().as_ref())
+                            .map(|tz| tz.as_utf8().as_ref());
+                        if time_zone_is_without_dst(time_zone) {
+                            any
+                        } else {
+                            Inherent(Unknown)
+                        }
+                    }
+                    _ => Inherent(Unknown),
+                },
+                _ => unreachable!(),
+            },
+            ExprType::AddWithTimeZone | ExprType::SubtractWithTimeZone => {
+                // Requires time zone and interval to be literal, at least for now.
+                let time_zone = match &func_call.inputs()[2] {
+                    ExprImpl::Literal(lit) => {
+                        lit.get_data().as_ref().map(|tz| tz.as_utf8().as_ref())
+                    }
+                    _ => return Inherent(Unknown),
+                };
+                let interval = match &func_call.inputs()[1] {
+                    ExprImpl::Literal(lit) => lit
+                        .get_data()
+                        .as_ref()
+                        .map(|interval| interval.as_interval()),
+                    _ => return Inherent(Unknown),
+                };
+                let quantitative_only = interval.map_or(
+                    true, // null interval is treated as `interval '1' second`
+                    |v| v.months() == 0 && (v.days() == 0 || time_zone_is_without_dst(time_zone)),
+                );
+                match (self.visit_expr(&func_call.inputs()[0]), quantitative_only) {
+                    (Inherent(Constant), _) => Inherent(Constant),
+                    (any, true) => any,
+                    _ => Inherent(Unknown),
+                }
+            }
+            ExprType::SecToTimestamptz => self.visit_unary_op(func_call.inputs()),
+            ExprType::CharToTimestamptz => Inherent(Unknown),
+            ExprType::Cast => {
+                // TODO: need more derivation
+                Inherent(Unknown)
+            }
+            ExprType::Case => {
+                // TODO: do we need derive watermark when every case can derive a common watermark?
+                Inherent(Unknown)
+            }
+            ExprType::Proctime => Inherent(NonDecreasing),
+            _ => Inherent(Unknown),
+        }
+    }
+
+    fn visit_table_function(&self, _table_func: &TableFunction) -> MonotonicityDerivation {
+        // TODO: derive monotonicity for table funcs like `generate_series`
+        use monotonicity_variants::*;
+        Inherent(Unknown)
+    }
+}
+
+/// A map from column index to its monotonicity.
+#[derive(Debug, Default, Clone, PartialEq, Eq, Hash)]
+pub struct MonotonicityMap(BTreeMap<usize, Monotonicity>);
+
+impl MonotonicityMap {
+    pub fn new() -> Self {
+        MonotonicityMap(BTreeMap::new())
+    }
+
+    pub fn insert(&mut self, idx: usize, monotonicity: Monotonicity) {
+        if monotonicity != Monotonicity::Unknown {
+            self.0.insert(idx, monotonicity);
+        }
+    }
+
+    pub fn iter(&self) -> impl Iterator<Item = (usize, Monotonicity)> + '_ {
+        self.0
+            .iter()
+            .map(|(idx, monotonicity)| (*idx, *monotonicity))
+    }
+}
+
+impl Index<usize> for MonotonicityMap {
+    type Output = Monotonicity;
+
+    fn index(&self, idx: usize) -> &Self::Output {
+        self.0.get(&idx).unwrap_or(&Monotonicity::Unknown)
+    }
+}
+
+impl IntoIterator for MonotonicityMap {
+    type IntoIter = std::collections::btree_map::IntoIter<usize, Monotonicity>;
+    type Item = (usize, Monotonicity);
+
+    fn into_iter(self) -> Self::IntoIter {
+        self.0.into_iter()
+    }
+}
+
+impl FromIterator<(usize, Monotonicity)> for MonotonicityMap {
+    fn from_iter<T: IntoIterator<Item = (usize, Monotonicity)>>(iter: T) -> Self {
+        MonotonicityMap(iter.into_iter().collect())
+    }
+}
diff --git a/src/frontend/src/optimizer/rule/agg_group_by_simplify_rule.rs b/src/frontend/src/optimizer/rule/agg_group_by_simplify_rule.rs
index d55c00d10bb2d..f7bc29611b618 100644
--- a/src/frontend/src/optimizer/rule/agg_group_by_simplify_rule.rs
+++ b/src/frontend/src/optimizer/rule/agg_group_by_simplify_rule.rs
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use risingwave_expr::aggregate::AggKind;
+use risingwave_expr::aggregate::PbAggKind;
 
 use super::super::plan_node::*;
 use super::{BoxedRule, Rule};
@@ -47,14 +47,13 @@ impl Rule for AggGroupBySimplifyRule {
                 if !new_group_key.contains(i) {
                     let data_type = agg_input.schema().fields[i].data_type();
                     new_agg_calls.push(PlanAggCall {
-                        agg_kind: AggKind::InternalLastSeenValue,
+                        agg_kind: PbAggKind::InternalLastSeenValue.into(),
                         return_type: data_type.clone(),
                         inputs: vec![InputRef::new(i, data_type)],
                         distinct: false,
                         order_by: vec![],
                         filter: Condition::true_cond(),
                         direct_args: vec![],
-                        user_defined: None,
                     });
                 }
             }
diff --git a/src/frontend/src/optimizer/rule/apply_agg_transpose_rule.rs b/src/frontend/src/optimizer/rule/apply_agg_transpose_rule.rs
index 48c3366d1d825..33bb59e59bf13 100644
--- a/src/frontend/src/optimizer/rule/apply_agg_transpose_rule.rs
+++ b/src/frontend/src/optimizer/rule/apply_agg_transpose_rule.rs
@@ -13,7 +13,7 @@
 // limitations under the License.
 
 use risingwave_common::types::DataType;
-use risingwave_expr::aggregate::AggKind;
+use risingwave_expr::aggregate::{AggKind, PbAggKind};
 use risingwave_pb::plan_common::JoinType;
 
 use super::{ApplyOffsetRewriter, BoxedRule, Rule};
@@ -141,49 +141,54 @@ impl Rule for ApplyAggTransposeRule {
                 let pos_of_constant_column = node.schema().len() - 1;
                 agg_calls.iter_mut().for_each(|agg_call| {
                     match agg_call.agg_kind {
-                        AggKind::Count if agg_call.inputs.is_empty() => {
+                        AggKind::Builtin(PbAggKind::Count) if agg_call.inputs.is_empty() => {
                             let input_ref = InputRef::new(pos_of_constant_column, DataType::Int32);
                             agg_call.inputs.push(input_ref);
                         }
-                        AggKind::ArrayAgg
-                        | AggKind::JsonbAgg
-                        | AggKind::JsonbObjectAgg
-                        | AggKind::UserDefined => {
+                        AggKind::Builtin(PbAggKind::ArrayAgg
+                        | PbAggKind::JsonbAgg
+                        | PbAggKind::JsonbObjectAgg)
+                        | AggKind::UserDefined(_)
+                        | AggKind::WrapScalar(_) => {
                             let input_ref = InputRef::new(pos_of_constant_column, DataType::Int32);
                             let cond = FunctionCall::new(ExprType::IsNotNull, vec![input_ref.into()]).unwrap();
                             agg_call.filter.conjunctions.push(cond.into());
                         }
-                        AggKind::Count
-                        | AggKind::Sum
-                        | AggKind::Sum0
-                        | AggKind::Avg
-                        | AggKind::Min
-                        | AggKind::Max
-                        | AggKind::BitAnd
-                        | AggKind::BitOr
-                        | AggKind::BitXor
-                        | AggKind::BoolAnd
-                        | AggKind::BoolOr
-                        | AggKind::StringAgg
+                        AggKind::Builtin(PbAggKind::Count
+                        | PbAggKind::Sum
+                        | PbAggKind::Sum0
+                        | PbAggKind::Avg
+                        | PbAggKind::Min
+                        | PbAggKind::Max
+                        | PbAggKind::BitAnd
+                        | PbAggKind::BitOr
+                        | PbAggKind::BitXor
+                        | PbAggKind::BoolAnd
+                        | PbAggKind::BoolOr
+                        | PbAggKind::StringAgg
                         // not in PostgreSQL
-                        | AggKind::ApproxCountDistinct
-                        | AggKind::FirstValue
-                        | AggKind::LastValue
-                        | AggKind::InternalLastSeenValue
+                        | PbAggKind::ApproxCountDistinct
+                        | PbAggKind::FirstValue
+                        | PbAggKind::LastValue
+                        | PbAggKind::InternalLastSeenValue
                         // All statistical aggregates only consider non-null inputs.
-                        | AggKind::VarPop
-                        | AggKind::VarSamp
-                        | AggKind::StddevPop
-                        | AggKind::StddevSamp
+                        | PbAggKind::ApproxPercentile
+                        | PbAggKind::VarPop
+                        | PbAggKind::VarSamp
+                        | PbAggKind::StddevPop
+                        | PbAggKind::StddevSamp
                         // All ordered-set aggregates ignore null values in their aggregated input.
-                        | AggKind::PercentileCont
-                        | AggKind::PercentileDisc
-                        | AggKind::Mode
+                        | PbAggKind::PercentileCont
+                        | PbAggKind::PercentileDisc
+                        | PbAggKind::Mode
                         // `grouping` has no *aggregate* input and unreachable when `is_scalar_agg`.
-                        | AggKind::Grouping
+                        | PbAggKind::Grouping)
                         => {
                             // no-op when `agg(0 rows) == agg(1 row of nulls)`
                         }
+                        AggKind::Builtin(PbAggKind::Unspecified | PbAggKind::UserDefined | PbAggKind::WrapScalar) => {
+                            panic!("Unexpected aggregate function: {:?}", agg_call.agg_kind)
+                        }
                     }
                 });
             }
diff --git a/src/frontend/src/optimizer/rule/distinct_agg_rule.rs b/src/frontend/src/optimizer/rule/distinct_agg_rule.rs
index 478b0ffb09cd6..52aad1336a7bb 100644
--- a/src/frontend/src/optimizer/rule/distinct_agg_rule.rs
+++ b/src/frontend/src/optimizer/rule/distinct_agg_rule.rs
@@ -18,7 +18,7 @@ use std::mem;
 use fixedbitset::FixedBitSet;
 use itertools::Itertools;
 use risingwave_common::types::DataType;
-use risingwave_expr::aggregate::{agg_kinds, AggKind};
+use risingwave_expr::aggregate::{agg_kinds, AggKind, PbAggKind};
 
 use super::{BoxedRule, Rule};
 use crate::expr::{CollectInputRef, ExprType, FunctionCall, InputRef, Literal};
@@ -57,8 +57,11 @@ impl Rule for DistinctAggRule {
                 c.agg_kind
             );
             let agg_kind_ok = !matches!(c.agg_kind, agg_kinds::simply_cannot_two_phase!());
-            let order_ok = matches!(c.agg_kind, agg_kinds::result_unaffected_by_order_by!())
-                || c.order_by.is_empty();
+            let order_ok = matches!(
+                c.agg_kind,
+                agg_kinds::result_unaffected_by_order_by!()
+                    | AggKind::Builtin(PbAggKind::ApproxPercentile)
+            ) || c.order_by.is_empty();
             agg_kind_ok && order_ok
         }) {
             tracing::warn!("DistinctAggRule: unsupported agg kind, fallback to backend impl");
diff --git a/src/frontend/src/optimizer/rule/grouping_sets_to_expand_rule.rs b/src/frontend/src/optimizer/rule/grouping_sets_to_expand_rule.rs
index 5bb1860df0d35..c24ab4e806c32 100644
--- a/src/frontend/src/optimizer/rule/grouping_sets_to_expand_rule.rs
+++ b/src/frontend/src/optimizer/rule/grouping_sets_to_expand_rule.rs
@@ -16,7 +16,7 @@ use fixedbitset::FixedBitSet;
 use itertools::Itertools;
 use risingwave_common::types::DataType;
 use risingwave_common::util::column_index_mapping::ColIndexMapping;
-use risingwave_expr::aggregate::AggKind;
+use risingwave_expr::aggregate::{AggKind, PbAggKind};
 
 use super::super::plan_node::*;
 use super::{BoxedRule, Rule};
@@ -105,7 +105,7 @@ impl Rule for GroupingSetsToExpandRule {
         let mut new_agg_calls = vec![];
         for agg_call in old_agg_calls {
             // Deal with grouping agg call for grouping sets.
-            if agg_call.agg_kind == AggKind::Grouping {
+            if matches!(agg_call.agg_kind, AggKind::Builtin(PbAggKind::Grouping)) {
                 let mut grouping_values = vec![];
                 let args = agg_call
                     .inputs
diff --git a/src/frontend/src/optimizer/rule/index_selection_rule.rs b/src/frontend/src/optimizer/rule/index_selection_rule.rs
index d47ad04de2763..e65b249379750 100644
--- a/src/frontend/src/optimizer/rule/index_selection_rule.rs
+++ b/src/frontend/src/optimizer/rule/index_selection_rule.rs
@@ -58,6 +58,7 @@ use risingwave_common::types::{
 };
 use risingwave_common::util::iter_util::ZipEqFast;
 use risingwave_pb::plan_common::JoinType;
+use risingwave_sqlparser::ast::AsOf;
 
 use super::{BoxedRule, Rule};
 use crate::catalog::IndexCatalog;
@@ -95,9 +96,6 @@ impl Rule for IndexSelectionRule {
         if indexes.is_empty() {
             return None;
         }
-        if logical_scan.as_of().is_some() {
-            return None;
-        }
         let primary_table_row_size = TableScanIoEstimator::estimate_row_size(logical_scan);
         let primary_cost = min(
             self.estimate_table_scan_cost(logical_scan, primary_table_row_size),
@@ -230,7 +228,7 @@ impl IndexSelectionRule {
             index.index_table.clone(),
             vec![],
             logical_scan.ctx(),
-            None,
+            logical_scan.as_of().clone(),
             index.index_table.cardinality,
         );
 
@@ -239,7 +237,7 @@ impl IndexSelectionRule {
             index.primary_table.clone(),
             vec![],
             logical_scan.ctx(),
-            None,
+            logical_scan.as_of().clone(),
             index.primary_table.cardinality,
         );
 
@@ -338,7 +336,7 @@ impl IndexSelectionRule {
             logical_scan.table_catalog(),
             vec![],
             logical_scan.ctx(),
-            None,
+            logical_scan.as_of().clone(),
             logical_scan.table_cardinality(),
         );
 
@@ -543,9 +541,12 @@ impl IndexSelectionRule {
                 let condition = Condition {
                     conjunctions: conj.iter().map(|&x| x.to_owned()).collect(),
                 };
-                if let Some(index_access) =
-                    self.build_index_access(index.clone(), condition, logical_scan.ctx().clone())
-                {
+                if let Some(index_access) = self.build_index_access(
+                    index.clone(),
+                    condition,
+                    logical_scan.ctx().clone(),
+                    logical_scan.as_of().clone(),
+                ) {
                     result.push(index_access);
                 }
             }
@@ -573,7 +574,7 @@ impl IndexSelectionRule {
             Condition {
                 conjunctions: conjunctions.to_vec(),
             },
-            None,
+            logical_scan.as_of().clone(),
             logical_scan.table_cardinality(),
         );
 
@@ -588,6 +589,7 @@ impl IndexSelectionRule {
         index: Rc<IndexCatalog>,
         predicate: Condition,
         ctx: OptimizerContextRef,
+        as_of: Option<AsOf>,
     ) -> Option<PlanRef> {
         let mut rewriter = IndexPredicateRewriter::new(
             index.primary_to_secondary_mapping(),
@@ -613,7 +615,7 @@ impl IndexSelectionRule {
                 vec![],
                 ctx,
                 new_predicate,
-                None,
+                as_of,
                 index.index_table.cardinality,
             )
             .into(),
diff --git a/src/frontend/src/optimizer/rule/min_max_on_index_rule.rs b/src/frontend/src/optimizer/rule/min_max_on_index_rule.rs
index 7c663ea714b6d..8782ca8f481ff 100644
--- a/src/frontend/src/optimizer/rule/min_max_on_index_rule.rs
+++ b/src/frontend/src/optimizer/rule/min_max_on_index_rule.rs
@@ -23,7 +23,7 @@ use std::vec;
 use itertools::Itertools;
 use risingwave_common::types::DataType;
 use risingwave_common::util::sort_util::{ColumnOrder, OrderType};
-use risingwave_expr::aggregate::AggKind;
+use risingwave_expr::aggregate::{AggKind, PbAggKind};
 
 use super::{BoxedRule, Rule};
 use crate::expr::{ExprImpl, ExprType, FunctionCall, InputRef};
@@ -49,20 +49,22 @@ impl Rule for MinMaxOnIndexRule {
         }
         let first_call = calls.iter().exactly_one().ok()?;
 
-        if matches!(first_call.agg_kind, AggKind::Min | AggKind::Max)
-            && !first_call.distinct
+        if matches!(
+            first_call.agg_kind,
+            AggKind::Builtin(PbAggKind::Min | PbAggKind::Max)
+        ) && !first_call.distinct
             && first_call.filter.always_true()
             && first_call.order_by.is_empty()
         {
             let logical_scan: LogicalScan = logical_agg.input().as_logical_scan()?.to_owned();
-            let kind = calls.first()?.agg_kind;
+            let kind = &calls.first()?.agg_kind;
             if !logical_scan.predicate().always_true() {
                 return None;
             }
             let order = Order {
                 column_orders: vec![ColumnOrder::new(
                     calls.first()?.inputs.first()?.index(),
-                    if kind == AggKind::Min {
+                    if matches!(kind, AggKind::Builtin(PbAggKind::Min)) {
                         OrderType::ascending()
                     } else {
                         OrderType::descending()
@@ -112,7 +114,7 @@ impl MinMaxOnIndexRule {
 
                 let formatting_agg = Agg::new(
                     vec![PlanAggCall {
-                        agg_kind: logical_agg.agg_calls().first()?.agg_kind,
+                        agg_kind: logical_agg.agg_calls().first()?.agg_kind.clone(),
                         return_type: logical_agg.schema().fields[0].data_type.clone(),
                         inputs: vec![InputRef::new(
                             0,
@@ -124,7 +126,6 @@ impl MinMaxOnIndexRule {
                             conjunctions: vec![],
                         },
                         direct_args: vec![],
-                        user_defined: None,
                     }],
                     IndexSet::empty(),
                     topn.into(),
@@ -183,7 +184,7 @@ impl MinMaxOnIndexRule {
 
             let formatting_agg = Agg::new(
                 vec![PlanAggCall {
-                    agg_kind: logical_agg.agg_calls().first()?.agg_kind,
+                    agg_kind: logical_agg.agg_calls().first()?.agg_kind.clone(),
                     return_type: logical_agg.schema().fields[0].data_type.clone(),
                     inputs: vec![InputRef::new(
                         0,
@@ -195,7 +196,6 @@ impl MinMaxOnIndexRule {
                         conjunctions: vec![],
                     },
                     direct_args: vec![],
-                    user_defined: None,
                 }],
                 IndexSet::empty(),
                 topn.into(),
diff --git a/src/frontend/src/optimizer/rule/mod.rs b/src/frontend/src/optimizer/rule/mod.rs
index fd06402c7497f..180dafa0c79b6 100644
--- a/src/frontend/src/optimizer/rule/mod.rs
+++ b/src/frontend/src/optimizer/rule/mod.rs
@@ -160,12 +160,14 @@ pub use agg_call_merge_rule::*;
 mod pull_up_correlated_predicate_agg_rule;
 mod source_to_iceberg_scan_rule;
 mod source_to_kafka_scan_rule;
+mod table_function_to_file_scan_rule;
 mod values_extract_project_rule;
 
 pub use batch::batch_push_limit_to_scan_rule::*;
 pub use pull_up_correlated_predicate_agg_rule::*;
 pub use source_to_iceberg_scan_rule::*;
 pub use source_to_kafka_scan_rule::*;
+pub use table_function_to_file_scan_rule::*;
 pub use values_extract_project_rule::*;
 
 #[macro_export]
@@ -228,6 +230,7 @@ macro_rules! for_all_rules {
             , { CrossJoinEliminateRule }
             , { ApplyTopNTransposeRule }
             , { TableFunctionToProjectSetRule }
+            , { TableFunctionToFileScanRule }
             , { ApplyLimitTransposeRule }
             , { CommonSubExprExtractRule }
             , { BatchProjectMergeRule }
diff --git a/src/frontend/src/optimizer/rule/over_window_to_agg_and_join_rule.rs b/src/frontend/src/optimizer/rule/over_window_to_agg_and_join_rule.rs
index 279d495e4681c..259d8333aea78 100644
--- a/src/frontend/src/optimizer/rule/over_window_to_agg_and_join_rule.rs
+++ b/src/frontend/src/optimizer/rule/over_window_to_agg_and_join_rule.rs
@@ -51,9 +51,9 @@ impl Rule for OverWindowToAggAndJoinRule {
             .collect_vec();
         let mut select_exprs = group_exprs.clone();
         for func in window_functions {
-            if let WindowFuncKind::Aggregate(kind) = func.kind {
+            if let WindowFuncKind::Aggregate(kind) = &func.kind {
                 let agg_call = AggCall::new(
-                    kind,
+                    kind.clone(),
                     func.args.iter().map(|x| x.clone().into()).collect_vec(),
                     false,
                     OrderBy::any(),
diff --git a/src/frontend/src/optimizer/rule/pull_up_correlated_predicate_agg_rule.rs b/src/frontend/src/optimizer/rule/pull_up_correlated_predicate_agg_rule.rs
index c9cb4edc860cd..e136c96c6e086 100644
--- a/src/frontend/src/optimizer/rule/pull_up_correlated_predicate_agg_rule.rs
+++ b/src/frontend/src/optimizer/rule/pull_up_correlated_predicate_agg_rule.rs
@@ -16,7 +16,7 @@ use fixedbitset::FixedBitSet;
 use itertools::{Either, Itertools};
 use risingwave_common::types::DataType;
 use risingwave_common::util::column_index_mapping::ColIndexMapping;
-use risingwave_expr::aggregate::AggKind;
+use risingwave_expr::aggregate::{AggKind, PbAggKind};
 
 use super::super::plan_node::*;
 use super::{BoxedRule, Rule};
@@ -162,13 +162,15 @@ impl Rule for PullUpCorrelatedPredicateAggRule {
         // sum is null, so avg is null. And null-rejected expression will be false, so we can still apply this rule and we don't need to generate a 0 value for count.
         let count_exists = agg_calls
             .iter()
-            .any(|agg_call| agg_call.agg_kind == AggKind::Count);
+            .any(|agg_call| matches!(agg_call.agg_kind, AggKind::Builtin(PbAggKind::Count)));
 
         if count_exists {
             // When group input is empty, not count agg would return null.
             let null_agg_pos = agg_calls
                 .iter()
-                .positions(|agg_call| agg_call.agg_kind != AggKind::Count)
+                .positions(|agg_call| {
+                    !matches!(agg_call.agg_kind, AggKind::Builtin(PbAggKind::Count))
+                })
                 .collect_vec();
 
             // If no null agg, bail out.
diff --git a/src/frontend/src/optimizer/rule/stream/filter_with_now_to_join_rule.rs b/src/frontend/src/optimizer/rule/stream/filter_with_now_to_join_rule.rs
index cbdb65b4528a5..39dc88319816d 100644
--- a/src/frontend/src/optimizer/rule/stream/filter_with_now_to_join_rule.rs
+++ b/src/frontend/src/optimizer/rule/stream/filter_with_now_to_join_rule.rs
@@ -15,11 +15,10 @@
 use risingwave_common::types::DataType;
 use risingwave_pb::plan_common::JoinType;
 
-use crate::expr::{
-    try_derive_watermark, ExprRewriter, FunctionCall, InputRef, WatermarkDerivation,
-};
+use crate::expr::{ExprRewriter, FunctionCall, InputRef};
 use crate::optimizer::plan_node::generic::{self, GenericPlanRef};
 use crate::optimizer::plan_node::{LogicalFilter, LogicalJoin, LogicalNow};
+use crate::optimizer::property::{analyze_monotonicity, monotonicity_variants};
 use crate::optimizer::rule::{BoxedRule, Rule};
 use crate::optimizer::PlanRef;
 use crate::utils::Condition;
@@ -36,18 +35,19 @@ impl Rule for FilterWithNowToJoinRule {
         let mut now_filters = vec![];
         let mut remainder = vec![];
 
-        let mut rewriter = NowAsInputRef::new(lhs_len);
-
         // If the `now` is not a valid dynamic filter expression, we will not push it down.
         filter.predicate().conjunctions.iter().for_each(|expr| {
             if let Some((input_expr, cmp, now_expr)) = expr.as_now_comparison_cond() {
-                let now_expr = rewriter.rewrite_expr(now_expr);
-
-                // ensure that this expression will derive a watermark
-                if try_derive_watermark(&now_expr) != WatermarkDerivation::Watermark(lhs_len) {
-                    remainder.push(expr.clone());
+                // ensure that this expression is increasing
+                use monotonicity_variants::*;
+                if matches!(analyze_monotonicity(&now_expr), Inherent(NonDecreasing)) {
+                    now_filters.push(
+                        FunctionCall::new(cmp, vec![input_expr, now_expr])
+                            .unwrap()
+                            .into(),
+                    );
                 } else {
-                    now_filters.push(FunctionCall::new(cmp, vec![input_expr, now_expr]).unwrap());
+                    remainder.push(expr.clone());
                 }
             } else {
                 remainder.push(expr.clone());
@@ -60,13 +60,15 @@ impl Rule for FilterWithNowToJoinRule {
         }
         let mut new_plan = plan.inputs()[0].clone();
 
+        let mut rewriter = NowAsInputRef::new(lhs_len);
         for now_filter in now_filters {
+            let now_filter = rewriter.rewrite_expr(now_filter);
             new_plan = LogicalJoin::new(
                 new_plan,
                 LogicalNow::new(generic::Now::update_current(plan.ctx())).into(),
                 JoinType::LeftSemi,
                 Condition {
-                    conjunctions: vec![now_filter.into()],
+                    conjunctions: vec![now_filter],
                 },
             )
             .into()
diff --git a/src/frontend/src/optimizer/rule/table_function_to_file_scan_rule.rs b/src/frontend/src/optimizer/rule/table_function_to_file_scan_rule.rs
new file mode 100644
index 0000000000000..63a4f894319fc
--- /dev/null
+++ b/src/frontend/src/optimizer/rule/table_function_to_file_scan_rule.rs
@@ -0,0 +1,90 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use itertools::Itertools;
+use risingwave_common::catalog::{Field, Schema};
+use risingwave_common::types::{DataType, ScalarImpl};
+use risingwave_common::util::iter_util::ZipEqDebug;
+
+use super::{BoxedRule, Rule};
+use crate::expr::{Expr, TableFunctionType};
+use crate::optimizer::plan_node::generic::GenericPlanRef;
+use crate::optimizer::plan_node::{LogicalFileScan, LogicalTableFunction};
+use crate::optimizer::PlanRef;
+
+/// Transform a special `TableFunction` (with `FILE_SCAN` table function type) into a `LogicalFileScan`
+pub struct TableFunctionToFileScanRule {}
+impl Rule for TableFunctionToFileScanRule {
+    fn apply(&self, plan: PlanRef) -> Option<PlanRef> {
+        let logical_table_function: &LogicalTableFunction = plan.as_logical_table_function()?;
+        if logical_table_function.table_function.function_type != TableFunctionType::FileScan {
+            return None;
+        }
+        assert!(!logical_table_function.with_ordinality);
+        let table_function_return_type = logical_table_function.table_function().return_type();
+
+        if let DataType::Struct(st) = table_function_return_type.clone() {
+            let fields = st
+                .types()
+                .zip_eq_debug(st.names())
+                .map(|(data_type, name)| Field::with_name(data_type.clone(), name.to_string()))
+                .collect_vec();
+
+            let schema = Schema::new(fields);
+
+            assert!(logical_table_function.table_function().args.len() >= 6);
+            let mut eval_args = vec![];
+            for arg in &logical_table_function.table_function().args {
+                assert_eq!(arg.return_type(), DataType::Varchar);
+                let value = arg.try_fold_const().unwrap().unwrap();
+                match value {
+                    Some(ScalarImpl::Utf8(s)) => {
+                        eval_args.push(s.to_string());
+                    }
+                    _ => {
+                        unreachable!("must be a varchar")
+                    }
+                }
+            }
+            assert!("parquet".eq_ignore_ascii_case(&eval_args[0]));
+            assert!("s3".eq_ignore_ascii_case(&eval_args[1]));
+            let s3_region = eval_args[2].clone();
+            let s3_access_key = eval_args[3].clone();
+            let s3_secret_key = eval_args[4].clone();
+            // The rest of the arguments are file locations
+            let file_location = eval_args[5..].iter().cloned().collect_vec();
+            Some(
+                LogicalFileScan::new(
+                    logical_table_function.ctx(),
+                    schema,
+                    "parquet".to_string(),
+                    "s3".to_string(),
+                    s3_region,
+                    s3_access_key,
+                    s3_secret_key,
+                    file_location,
+                )
+                .into(),
+            )
+        } else {
+            unreachable!("TableFunction return type should be struct")
+        }
+    }
+}
+
+impl TableFunctionToFileScanRule {
+    pub fn create() -> BoxedRule {
+        Box::new(TableFunctionToFileScanRule {})
+    }
+}
diff --git a/src/frontend/src/planner/query.rs b/src/frontend/src/planner/query.rs
index 2edde6105d190..bbf152c36fdfa 100644
--- a/src/frontend/src/planner/query.rs
+++ b/src/frontend/src/planner/query.rs
@@ -25,6 +25,8 @@ pub const LIMIT_ALL_COUNT: u64 = u64::MAX / 2;
 
 impl Planner {
     /// Plan a [`BoundQuery`]. Need to bind before planning.
+    ///
+    /// Works for both batch query and streaming query (`CREATE MATERIALIZED VIEW`).
     pub fn plan_query(&mut self, query: BoundQuery) -> Result<PlanRoot> {
         let out_names = query.schema().names();
         let BoundQuery {
diff --git a/src/frontend/src/planner/relation.rs b/src/frontend/src/planner/relation.rs
index 7941d2837a068..b5f8d39276d24 100644
--- a/src/frontend/src/planner/relation.rs
+++ b/src/frontend/src/planner/relation.rs
@@ -74,10 +74,11 @@ impl Planner {
     pub(super) fn plan_base_table(&mut self, base_table: &BoundBaseTable) -> Result<PlanRef> {
         let as_of = base_table.as_of.clone();
         match as_of {
-            None | Some(AsOf::ProcessTime) => {}
-            Some(AsOf::TimestampString(_)) | Some(AsOf::TimestampNum(_)) => {
-                bail_not_implemented!("As Of Timestamp is not supported yet.")
-            }
+            None
+            | Some(AsOf::ProcessTime)
+            | Some(AsOf::TimestampNum(_))
+            | Some(AsOf::TimestampString(_))
+            | Some(AsOf::ProcessTimeWithInterval(_)) => {}
             Some(AsOf::VersionNum(_)) | Some(AsOf::VersionString(_)) => {
                 bail_not_implemented!("As Of Version is not supported yet.")
             }
@@ -111,7 +112,7 @@ impl Planner {
                 | Some(AsOf::VersionNum(_))
                 | Some(AsOf::TimestampString(_))
                 | Some(AsOf::TimestampNum(_)) => {}
-                Some(AsOf::ProcessTime) => {
+                Some(AsOf::ProcessTime) | Some(AsOf::ProcessTimeWithInterval(_)) => {
                     bail_not_implemented!("As Of ProcessTime() is not supported yet.")
                 }
                 Some(AsOf::VersionString(_)) => {
diff --git a/src/frontend/src/planner/set_expr.rs b/src/frontend/src/planner/set_expr.rs
index e2ff43a2c211b..8c8405b039b91 100644
--- a/src/frontend/src/planner/set_expr.rs
+++ b/src/frontend/src/planner/set_expr.rs
@@ -34,9 +34,10 @@ impl Planner {
             BoundSetExpr::SetOperation {
                 op,
                 all,
+                corresponding_col_indices,
                 left,
                 right,
-            } => self.plan_set_operation(op, all, *left, *right),
+            } => self.plan_set_operation(op, all, corresponding_col_indices, *left, *right),
         }
     }
 }
diff --git a/src/frontend/src/planner/set_operation.rs b/src/frontend/src/planner/set_operation.rs
index 1050c28bd11fb..8abdac8a7e2b1 100644
--- a/src/frontend/src/planner/set_operation.rs
+++ b/src/frontend/src/planner/set_operation.rs
@@ -1,3 +1,5 @@
+use risingwave_common::util::column_index_mapping::ColIndexMapping;
+
 // Copyright 2024 RisingWave Labs
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,7 +15,7 @@
 // limitations under the License.
 use crate::binder::{BoundSetExpr, BoundSetOperation};
 use crate::error::Result;
-use crate::optimizer::plan_node::{LogicalExcept, LogicalIntersect, LogicalUnion};
+use crate::optimizer::plan_node::{LogicalExcept, LogicalIntersect, LogicalProject, LogicalUnion};
 use crate::planner::Planner;
 use crate::PlanRef;
 
@@ -22,25 +24,27 @@ impl Planner {
         &mut self,
         op: BoundSetOperation,
         all: bool,
+        corresponding_col_indices: Option<(ColIndexMapping, ColIndexMapping)>,
         left: BoundSetExpr,
         right: BoundSetExpr,
     ) -> Result<PlanRef> {
+        let left = self.plan_set_expr(left, vec![], &[])?;
+        let right = self.plan_set_expr(right, vec![], &[])?;
+
+        // Map the corresponding columns
+        let (left, right) = if let Some((mapping_l, mapping_r)) = corresponding_col_indices {
+            (
+                LogicalProject::with_mapping(left, mapping_l).into(),
+                LogicalProject::with_mapping(right, mapping_r).into(),
+            )
+        } else {
+            (left, right)
+        };
+
         match op {
-            BoundSetOperation::Union => {
-                let left = self.plan_set_expr(left, vec![], &[])?;
-                let right = self.plan_set_expr(right, vec![], &[])?;
-                Ok(LogicalUnion::create(all, vec![left, right]))
-            }
-            BoundSetOperation::Intersect => {
-                let left = self.plan_set_expr(left, vec![], &[])?;
-                let right = self.plan_set_expr(right, vec![], &[])?;
-                Ok(LogicalIntersect::create(all, vec![left, right]))
-            }
-            BoundSetOperation::Except => {
-                let left = self.plan_set_expr(left, vec![], &[])?;
-                let right = self.plan_set_expr(right, vec![], &[])?;
-                Ok(LogicalExcept::create(all, vec![left, right]))
-            }
+            BoundSetOperation::Union => Ok(LogicalUnion::create(all, vec![left, right])),
+            BoundSetOperation::Intersect => Ok(LogicalIntersect::create(all, vec![left, right])),
+            BoundSetOperation::Except => Ok(LogicalExcept::create(all, vec![left, right])),
         }
     }
 }
diff --git a/src/frontend/src/planner/statement.rs b/src/frontend/src/planner/statement.rs
index 0eed65e2df7e6..22b63de9f40be 100644
--- a/src/frontend/src/planner/statement.rs
+++ b/src/frontend/src/planner/statement.rs
@@ -24,6 +24,7 @@ impl Planner {
             BoundStatement::Delete(d) => self.plan_delete(*d),
             BoundStatement::Update(u) => self.plan_update(*u),
             BoundStatement::Query(q) => self.plan_query(*q),
+            BoundStatement::CreateView(c) => self.plan_query(*c.query),
         }
     }
 }
diff --git a/src/frontend/src/scheduler/distributed/query.rs b/src/frontend/src/scheduler/distributed/query.rs
index 5ab5f1aa85af7..ce13960ab221f 100644
--- a/src/frontend/src/scheduler/distributed/query.rs
+++ b/src/frontend/src/scheduler/distributed/query.rs
@@ -479,7 +479,7 @@ pub(crate) mod tests {
     use risingwave_common::hash::{WorkerSlotId, WorkerSlotMapping};
     use risingwave_common::types::DataType;
     use risingwave_pb::common::worker_node::Property;
-    use risingwave_pb::common::{HostAddress, ParallelUnit, WorkerNode, WorkerType};
+    use risingwave_pb::common::{HostAddress, WorkerNode, WorkerType};
     use risingwave_pb::plan_common::JoinType;
     use risingwave_rpc_client::ComputeClientPool;
 
@@ -507,7 +507,7 @@ pub(crate) mod tests {
     async fn test_query_should_not_hang_with_empty_worker() {
         let worker_node_manager = Arc::new(WorkerNodeManager::mock(vec![]));
         let worker_node_selector = WorkerNodeSelector::new(worker_node_manager.clone(), false);
-        let compute_client_pool = Arc::new(ComputeClientPool::default());
+        let compute_client_pool = Arc::new(ComputeClientPool::for_test());
         let hummock_snapshot_manager = Arc::new(HummockSnapshotManager::new(Arc::new(
             MockFrontendMetaClient {},
         )));
@@ -675,7 +675,7 @@ pub(crate) mod tests {
                 port: 5687,
             }),
             state: risingwave_pb::common::worker_node::State::Running as i32,
-            parallel_units: generate_parallel_units(0, 0),
+            parallelism: 8,
             property: Some(Property {
                 is_unschedulable: false,
                 is_serving: true,
@@ -692,7 +692,7 @@ pub(crate) mod tests {
                 port: 5688,
             }),
             state: risingwave_pb::common::worker_node::State::Running as i32,
-            parallel_units: generate_parallel_units(8, 1),
+            parallelism: 8,
             property: Some(Property {
                 is_unschedulable: false,
                 is_serving: true,
@@ -709,7 +709,7 @@ pub(crate) mod tests {
                 port: 5689,
             }),
             state: risingwave_pb::common::worker_node::State::Running as i32,
-            parallel_units: generate_parallel_units(16, 2),
+            parallelism: 8,
             property: Some(Property {
                 is_unschedulable: false,
                 is_serving: true,
@@ -743,14 +743,4 @@ pub(crate) mod tests {
         .unwrap();
         fragmenter.generate_complete_query().await.unwrap()
     }
-
-    fn generate_parallel_units(start_id: u32, node_id: u32) -> Vec<ParallelUnit> {
-        let parallel_degree = 8;
-        (start_id..start_id + parallel_degree)
-            .map(|id| ParallelUnit {
-                id,
-                worker_node_id: node_id,
-            })
-            .collect()
-    }
 }
diff --git a/src/frontend/src/scheduler/distributed/stage.rs b/src/frontend/src/scheduler/distributed/stage.rs
index 211daca076ea1..543d0c0a3ae6f 100644
--- a/src/frontend/src/scheduler/distributed/stage.rs
+++ b/src/frontend/src/scheduler/distributed/stage.rs
@@ -405,6 +405,27 @@ impl StageRunner {
                     expr_context.clone(),
                 ));
             }
+        } else if let Some(file_scan_info) = self.stage.file_scan_info.as_ref() {
+            let chunk_size = (file_scan_info.file_location.len() as f32
+                / self.stage.parallelism.unwrap() as f32)
+                .ceil() as usize;
+            for (id, files) in file_scan_info.file_location.chunks(chunk_size).enumerate() {
+                let task_id = PbTaskId {
+                    query_id: self.stage.query_id.id.clone(),
+                    stage_id: self.stage.id,
+                    task_id: id as u64,
+                };
+                let plan_fragment =
+                    self.create_plan_fragment(id as u64, Some(PartitionInfo::File(files.to_vec())));
+                let worker =
+                    self.choose_worker(&plan_fragment, id as u32, self.stage.dml_table_id)?;
+                futures.push(self.schedule_task(
+                    task_id,
+                    plan_fragment,
+                    worker,
+                    expr_context.clone(),
+                ));
+            }
         } else {
             for id in 0..self.stage.parallelism.unwrap() {
                 let task_id = PbTaskId {
@@ -712,11 +733,11 @@ impl StageRunner {
 
         if let Some(table_id) = dml_table_id {
             let vnode_mapping = self.get_table_dml_vnode_mapping(&table_id)?;
-            let worker_ids = vnode_mapping.iter_unique().collect_vec();
+            let worker_slot_ids = vnode_mapping.iter_unique().collect_vec();
             let candidates = self
                 .worker_node_manager
                 .manager
-                .get_workers_by_worker_slot_ids(&worker_ids)?;
+                .get_workers_by_worker_slot_ids(&worker_slot_ids)?;
             if candidates.is_empty() {
                 return Err(BatchError::EmptyWorkerNodes.into());
             }
diff --git a/src/frontend/src/scheduler/local.rs b/src/frontend/src/scheduler/local.rs
index 4484d6283f71e..a7ff6eabdf7ff 100644
--- a/src/frontend/src/scheduler/local.rs
+++ b/src/frontend/src/scheduler/local.rs
@@ -150,6 +150,7 @@ impl LocalQueryExecution {
         let search_path = self.session.config().search_path();
         let time_zone = self.session.config().timezone();
         let timeout = self.timeout;
+        let meta_client = self.front_env.meta_client_ref();
 
         let sender1 = sender.clone();
         let exec = async move {
@@ -171,7 +172,7 @@ impl LocalQueryExecution {
         use risingwave_expr::expr_context::TIME_ZONE;
 
         use crate::expr::function_impl::context::{
-            AUTH_CONTEXT, CATALOG_READER, DB_NAME, SEARCH_PATH, USER_INFO_READER,
+            AUTH_CONTEXT, CATALOG_READER, DB_NAME, META_CLIENT, SEARCH_PATH, USER_INFO_READER,
         };
 
         // box is necessary, otherwise the size of `exec` will double each time it is nested.
@@ -181,6 +182,7 @@ impl LocalQueryExecution {
         let exec = async move { SEARCH_PATH::scope(search_path, exec).await }.boxed();
         let exec = async move { AUTH_CONTEXT::scope(auth_context, exec).await }.boxed();
         let exec = async move { TIME_ZONE::scope(time_zone, exec).await }.boxed();
+        let exec = async move { META_CLIENT::scope(meta_client, exec).await }.boxed();
 
         if let Some(timeout) = timeout {
             let exec = async move {
@@ -400,6 +402,45 @@ impl LocalQueryExecution {
                         };
                         sources.push(exchange_source);
                     }
+                } else if let Some(file_scan_info) = &second_stage.file_scan_info {
+                    let chunk_size = (file_scan_info.file_location.len() as f32
+                        / (self.worker_node_manager.schedule_unit_count()) as f32)
+                        .ceil() as usize;
+                    for (id, files) in file_scan_info.file_location.chunks(chunk_size).enumerate() {
+                        let second_stage_plan_node = self.convert_plan_node(
+                            &second_stage.root,
+                            &mut None,
+                            Some(PartitionInfo::File(files.to_vec())),
+                            next_executor_id.clone(),
+                        )?;
+                        let second_stage_plan_fragment = PlanFragment {
+                            root: Some(second_stage_plan_node),
+                            exchange_info: Some(ExchangeInfo {
+                                mode: DistributionMode::Single as i32,
+                                ..Default::default()
+                            }),
+                        };
+                        let local_execute_plan = LocalExecutePlan {
+                            plan: Some(second_stage_plan_fragment),
+                            epoch: Some(self.snapshot.batch_query_epoch()),
+                            tracing_context: tracing_context.clone(),
+                        };
+                        // NOTE: select a random work node here.
+                        let worker_node = self.worker_node_manager.next_random_worker()?;
+                        let exchange_source = ExchangeSource {
+                            task_output_id: Some(TaskOutputId {
+                                task_id: Some(PbTaskId {
+                                    task_id: id as u64,
+                                    stage_id: exchange_source_stage_id,
+                                    query_id: self.query.query_id.id.clone(),
+                                }),
+                                output_id: 0,
+                            }),
+                            host: Some(worker_node.host.as_ref().unwrap().clone()),
+                            local_execute_plan: Some(Plan(local_execute_plan)),
+                        };
+                        sources.push(exchange_source);
+                    }
                 } else {
                     let second_stage_plan_node = self.convert_plan_node(
                         &second_stage.root,
@@ -493,6 +534,26 @@ impl LocalQueryExecution {
                     node_body: Some(node_body),
                 })
             }
+            PlanNodeType::BatchFileScan => {
+                let mut node_body = execution_plan_node.node.clone();
+                match &mut node_body {
+                    NodeBody::FileScan(ref mut file_scan_node) => {
+                        if let Some(partition) = partition {
+                            let partition = partition
+                                .into_file()
+                                .expect("PartitionInfo should be FilePartitionInfo here");
+                            file_scan_node.file_location = partition;
+                        }
+                    }
+                    _ => unreachable!(),
+                }
+
+                Ok(PbPlanNode {
+                    children: vec![],
+                    identity,
+                    node_body: Some(node_body),
+                })
+            }
             PlanNodeType::BatchSource
             | PlanNodeType::BatchKafkaScan
             | PlanNodeType::BatchIcebergScan => {
@@ -530,7 +591,7 @@ impl LocalQueryExecution {
                             self.get_fragment_id(&side_table_desc.table_id.into())?,
                         )?;
 
-                        // TODO: should we use `pb::ParallelUnitMapping` here?
+                        // TODO: should we use `pb::WorkerSlotMapping` here?
                         node.inner_side_vnode_mapping =
                             mapping.to_expanded().into_iter().map(u64::from).collect();
                         node.worker_nodes = self.worker_node_manager.manager.list_worker_nodes();
diff --git a/src/frontend/src/scheduler/plan_fragmenter.rs b/src/frontend/src/scheduler/plan_fragmenter.rs
index 6b04f62e60f89..7643e5c5e7ba2 100644
--- a/src/frontend/src/scheduler/plan_fragmenter.rs
+++ b/src/frontend/src/scheduler/plan_fragmenter.rs
@@ -27,8 +27,8 @@ use pgwire::pg_server::SessionId;
 use risingwave_batch::error::BatchError;
 use risingwave_batch::worker_manager::worker_node_manager::WorkerNodeSelector;
 use risingwave_common::bail;
-use risingwave_common::buffer::{Bitmap, BitmapBuilder};
-use risingwave_common::catalog::TableDesc;
+use risingwave_common::bitmap::{Bitmap, BitmapBuilder};
+use risingwave_common::catalog::{Schema, TableDesc};
 use risingwave_common::hash::table_distribution::TableDistribution;
 use risingwave_common::hash::{VirtualNode, WorkerSlotId, WorkerSlotMapping};
 use risingwave_common::util::scan_range::ScanRange;
@@ -269,6 +269,7 @@ impl Query {
 
 #[derive(Debug, Clone)]
 pub struct SourceFetchInfo {
+    pub schema: Schema,
     pub connector: ConnectorProperties,
     pub timebound: (Option<i64>, Option<i64>),
     pub as_of: Option<AsOf>,
@@ -351,12 +352,14 @@ impl SourceScanInfo {
                             })
                             .map_err(|_e| anyhow!("fail to parse timestamp"))?,
                     ),
-                    Some(AsOf::ProcessTime) => unreachable!(),
+                    Some(AsOf::ProcessTime) | Some(AsOf::ProcessTimeWithInterval(_)) => {
+                        unreachable!()
+                    }
                     None => None,
                 };
 
                 let split_info = iceberg_enumerator
-                    .list_splits_batch(time_travel_info, batch_parallelism)
+                    .list_splits_batch(fetch_info.schema, time_travel_info, batch_parallelism)
                     .await?
                     .into_iter()
                     .map(SplitImpl::Iceberg)
@@ -431,6 +434,12 @@ pub struct TablePartitionInfo {
 pub enum PartitionInfo {
     Table(TablePartitionInfo),
     Source(Vec<SplitImpl>),
+    File(Vec<String>),
+}
+
+#[derive(Clone, Debug)]
+pub struct FileScanInfo {
+    pub file_location: Vec<String>,
 }
 
 /// Fragment part of `Query`.
@@ -444,6 +453,7 @@ pub struct QueryStage {
     /// Indicates whether this stage contains a table scan node and the table's information if so.
     pub table_scan_info: Option<TableScanInfo>,
     pub source_info: Option<SourceScanInfo>,
+    pub file_scan_info: Option<FileScanInfo>,
     pub has_lookup_join: bool,
     pub dml_table_id: Option<TableId>,
     pub session_id: SessionId,
@@ -467,16 +477,21 @@ impl QueryStage {
         self.has_lookup_join
     }
 
-    pub fn clone_with_exchange_info(&self, exchange_info: Option<ExchangeInfo>) -> Self {
+    pub fn clone_with_exchange_info(
+        &self,
+        exchange_info: Option<ExchangeInfo>,
+        parallelism: Option<u32>,
+    ) -> Self {
         if let Some(exchange_info) = exchange_info {
             return Self {
                 query_id: self.query_id.clone(),
                 id: self.id,
                 root: self.root.clone(),
                 exchange_info: Some(exchange_info),
-                parallelism: self.parallelism,
+                parallelism,
                 table_scan_info: self.table_scan_info.clone(),
                 source_info: self.source_info.clone(),
+                file_scan_info: self.file_scan_info.clone(),
                 has_lookup_join: self.has_lookup_join,
                 dml_table_id: self.dml_table_id,
                 session_id: self.session_id,
@@ -507,6 +522,7 @@ impl QueryStage {
             parallelism: Some(task_parallelism),
             table_scan_info: self.table_scan_info.clone(),
             source_info: Some(source_info),
+            file_scan_info: self.file_scan_info.clone(),
             has_lookup_join: self.has_lookup_join,
             dml_table_id: self.dml_table_id,
             session_id: self.session_id,
@@ -553,6 +569,7 @@ struct QueryStageBuilder {
     /// See also [`QueryStage::table_scan_info`].
     table_scan_info: Option<TableScanInfo>,
     source_info: Option<SourceScanInfo>,
+    file_scan_file: Option<FileScanInfo>,
     has_lookup_join: bool,
     dml_table_id: Option<TableId>,
     session_id: SessionId,
@@ -570,6 +587,7 @@ impl QueryStageBuilder {
         exchange_info: Option<ExchangeInfo>,
         table_scan_info: Option<TableScanInfo>,
         source_info: Option<SourceScanInfo>,
+        file_scan_file: Option<FileScanInfo>,
         has_lookup_join: bool,
         dml_table_id: Option<TableId>,
         session_id: SessionId,
@@ -584,6 +602,7 @@ impl QueryStageBuilder {
             children_stages: vec![],
             table_scan_info,
             source_info,
+            file_scan_file,
             has_lookup_join,
             dml_table_id,
             session_id,
@@ -606,6 +625,7 @@ impl QueryStageBuilder {
             parallelism: self.parallelism,
             table_scan_info: self.table_scan_info,
             source_info: self.source_info,
+            file_scan_info: self.file_scan_file,
             has_lookup_join: self.has_lookup_join,
             dml_table_id: self.dml_table_id,
             session_id: self.session_id,
@@ -698,15 +718,10 @@ impl StageGraph {
             // If the stage has parallelism, it means it's a complete stage.
             complete_stages.insert(
                 stage.id,
-                Arc::new(stage.clone_with_exchange_info(exchange_info)),
+                Arc::new(stage.clone_with_exchange_info(exchange_info, stage.parallelism)),
             );
             None
-        } else {
-            assert!(matches!(
-                stage.source_info,
-                Some(SourceScanInfo::Incomplete(_))
-            ));
-
+        } else if matches!(stage.source_info, Some(SourceScanInfo::Incomplete(_))) {
             let complete_source_info = stage
                 .source_info
                 .as_ref()
@@ -739,6 +754,17 @@ impl StageGraph {
             let parallelism = complete_stage.parallelism;
             complete_stages.insert(stage.id, complete_stage);
             parallelism
+        } else {
+            assert!(stage.file_scan_info.is_some());
+            let parallelism = min(
+                self.batch_parallelism / 2,
+                stage.file_scan_info.as_ref().unwrap().file_location.len(),
+            );
+            complete_stages.insert(
+                stage.id,
+                Arc::new(stage.clone_with_exchange_info(exchange_info, Some(parallelism as u32))),
+            );
+            None
         };
 
         for child_stage_id in self.child_edges.get(&stage.id).unwrap_or(&HashSet::new()) {
@@ -852,6 +878,13 @@ impl BatchPlanFragmenter {
         } else {
             None
         };
+
+        let file_scan_info = if table_scan_info.is_none() && source_info.is_none() {
+            Self::collect_stage_file_scan(root.clone())?
+        } else {
+            None
+        };
+
         let mut has_lookup_join = false;
         let parallelism = match root.distribution() {
             Distribution::Single => {
@@ -899,12 +932,14 @@ impl BatchPlanFragmenter {
                     lookup_join_parallelism
                 } else if source_info.is_some() {
                     0
+                } else if file_scan_info.is_some() {
+                    1
                 } else {
                     self.batch_parallelism
                 }
             }
         };
-        if source_info.is_none() && parallelism == 0 {
+        if source_info.is_none() && file_scan_info.is_none() && parallelism == 0 {
             return Err(BatchError::EmptyWorkerNodes.into());
         }
         let parallelism = if parallelism == 0 {
@@ -920,6 +955,7 @@ impl BatchPlanFragmenter {
             exchange_info,
             table_scan_info,
             source_info,
+            file_scan_info,
             has_lookup_join,
             dml_table_id,
             root.ctx().session_ctx().session_id(),
@@ -1009,12 +1045,11 @@ impl BatchPlanFragmenter {
             let batch_kafka_scan: &BatchKafkaScan = batch_kafka_node;
             let source_catalog = batch_kafka_scan.source_catalog();
             if let Some(source_catalog) = source_catalog {
-                let property = ConnectorProperties::extract(
-                    source_catalog.with_properties.clone().into_iter().collect(),
-                    false,
-                )?;
+                let property =
+                    ConnectorProperties::extract(source_catalog.with_properties.clone(), false)?;
                 let timestamp_bound = batch_kafka_scan.kafka_timestamp_range_value();
                 return Ok(Some(SourceScanInfo::new(SourceFetchInfo {
+                    schema: batch_kafka_scan.base.schema().clone(),
                     connector: property,
                     timebound: timestamp_bound,
                     as_of: None,
@@ -1024,12 +1059,11 @@ impl BatchPlanFragmenter {
             let batch_iceberg_scan: &BatchIcebergScan = batch_iceberg_scan;
             let source_catalog = batch_iceberg_scan.source_catalog();
             if let Some(source_catalog) = source_catalog {
-                let property = ConnectorProperties::extract(
-                    source_catalog.with_properties.clone().into_iter().collect(),
-                    false,
-                )?;
+                let property =
+                    ConnectorProperties::extract(source_catalog.with_properties.clone(), false)?;
                 let as_of = batch_iceberg_scan.as_of();
                 return Ok(Some(SourceScanInfo::new(SourceFetchInfo {
+                    schema: batch_iceberg_scan.base.schema().clone(),
                     connector: property,
                     timebound: (None, None),
                     as_of,
@@ -1040,12 +1074,11 @@ impl BatchPlanFragmenter {
             let source_node: &BatchSource = source_node;
             let source_catalog = source_node.source_catalog();
             if let Some(source_catalog) = source_catalog {
-                let property = ConnectorProperties::extract(
-                    source_catalog.with_properties.clone().into_iter().collect(),
-                    false,
-                )?;
+                let property =
+                    ConnectorProperties::extract(source_catalog.with_properties.clone(), false)?;
                 let as_of = source_node.as_of();
                 return Ok(Some(SourceScanInfo::new(SourceFetchInfo {
+                    schema: source_node.base.schema().clone(),
                     connector: property,
                     timebound: (None, None),
                     as_of,
@@ -1059,6 +1092,24 @@ impl BatchPlanFragmenter {
             .transpose()
     }
 
+    fn collect_stage_file_scan(node: PlanRef) -> SchedulerResult<Option<FileScanInfo>> {
+        if node.node_type() == PlanNodeType::BatchExchange {
+            // Do not visit next stage.
+            return Ok(None);
+        }
+
+        if let Some(batch_file_scan) = node.as_batch_file_scan() {
+            return Ok(Some(FileScanInfo {
+                file_location: batch_file_scan.core.file_location.clone(),
+            }));
+        }
+
+        node.inputs()
+            .into_iter()
+            .find_map(|n| Self::collect_stage_file_scan(n).transpose())
+            .transpose()
+    }
+
     /// Check whether this stage contains a table scan node and the table's information if so.
     ///
     /// If there are multiple scan nodes in this stage, they must have the same distribution, but
diff --git a/src/frontend/src/scheduler/snapshot.rs b/src/frontend/src/scheduler/snapshot.rs
index 36285b59b5e36..9d13573e03a7b 100644
--- a/src/frontend/src/scheduler/snapshot.rs
+++ b/src/frontend/src/scheduler/snapshot.rs
@@ -65,7 +65,7 @@ impl ReadSnapshot {
         match self.batch_query_epoch().epoch.unwrap() {
             batch_query_epoch::Epoch::Committed(epoch)
             | batch_query_epoch::Epoch::Current(epoch) => Some(epoch.into()),
-            batch_query_epoch::Epoch::Backup(_) => None,
+            batch_query_epoch::Epoch::Backup(_) | batch_query_epoch::Epoch::TimeTravel(_) => None,
         }
     }
 
@@ -74,7 +74,8 @@ impl ReadSnapshot {
         match self.batch_query_epoch().epoch.unwrap() {
             batch_query_epoch::Epoch::Committed(epoch)
             | batch_query_epoch::Epoch::Current(epoch)
-            | batch_query_epoch::Epoch::Backup(epoch) => epoch.into(),
+            | batch_query_epoch::Epoch::Backup(epoch)
+            | batch_query_epoch::Epoch::TimeTravel(epoch) => epoch.into(),
         }
     }
 
diff --git a/src/frontend/src/scheduler/task_context.rs b/src/frontend/src/scheduler/task_context.rs
index fff154d8a196e..a54c37debc38e 100644
--- a/src/frontend/src/scheduler/task_context.rs
+++ b/src/frontend/src/scheduler/task_context.rs
@@ -58,7 +58,6 @@ impl BatchTaskContext for FrontendBatchTaskContext {
         Arc::new(SysCatalogReaderImpl::new(
             self.session.env().catalog_reader().clone(),
             self.session.env().user_info_reader().clone(),
-            self.session.env().worker_node_manager_ref(),
             self.session.env().meta_client_ref(),
             self.session.auth_context(),
             self.session.shared_config(),
diff --git a/src/frontend/src/session.rs b/src/frontend/src/session.rs
index 9ac63eeb391af..4dd7aaa883061 100644
--- a/src/frontend/src/session.rs
+++ b/src/frontend/src/session.rs
@@ -51,6 +51,7 @@ use risingwave_common::config::{
     load_config, BatchConfig, MetaConfig, MetricLevel, StreamingConfig,
 };
 use risingwave_common::memory::MemoryContext;
+use risingwave_common::secret::LocalSecretManager;
 use risingwave_common::session_config::{ConfigReporter, SessionConfig, VisibilityMode};
 use risingwave_common::system_param::local_manager::{
     LocalSystemParamsManager, LocalSystemParamsManagerRef,
@@ -64,8 +65,7 @@ use risingwave_common::util::resource_util;
 use risingwave_common::util::runtime::BackgroundShutdownRuntime;
 use risingwave_common::{GIT_SHA, RW_VERSION};
 use risingwave_common_heap_profiling::HeapProfiler;
-use risingwave_common_service::observer_manager::ObserverManager;
-use risingwave_common_service::MetricsManager;
+use risingwave_common_service::{MetricsManager, ObserverManager};
 use risingwave_connector::source::monitor::{SourceMetrics, GLOBAL_SOURCE_METRICS};
 use risingwave_pb::common::WorkerType;
 use risingwave_pb::health::health_server::HealthServer;
@@ -86,6 +86,7 @@ use crate::binder::{Binder, BoundStatement, ResolveQualifiedNameError};
 use crate::catalog::catalog_service::{CatalogReader, CatalogWriter, CatalogWriterImpl};
 use crate::catalog::connection_catalog::ConnectionCatalog;
 use crate::catalog::root_catalog::Catalog;
+use crate::catalog::secret_catalog::SecretCatalog;
 use crate::catalog::subscription_catalog::SubscriptionCatalog;
 use crate::catalog::{
     check_schema_writable, CatalogError, DatabaseId, OwnedByUserCatalog, SchemaId, TableId,
@@ -187,7 +188,7 @@ impl FrontendEnv {
         let meta_client = Arc::new(MockFrontendMetaClient {});
         let hummock_snapshot_manager = Arc::new(HummockSnapshotManager::new(meta_client.clone()));
         let system_params_manager = Arc::new(LocalSystemParamsManager::for_test());
-        let compute_client_pool = Arc::new(ComputeClientPool::default());
+        let compute_client_pool = Arc::new(ComputeClientPool::for_test());
         let query_manager = QueryManager::new(
             worker_node_manager.clone(),
             compute_client_pool,
@@ -197,7 +198,7 @@ impl FrontendEnv {
             None,
         );
         let server_addr = HostAddr::try_from("127.0.0.1:4565").unwrap();
-        let client_pool = Arc::new(ComputeClientPool::default());
+        let client_pool = Arc::new(ComputeClientPool::for_test());
         let creating_streaming_tracker = StreamingJobTracker::new(meta_client.clone());
         let compute_runtime = Arc::new(BackgroundShutdownRuntime::from(
             Builder::new_multi_thread()
@@ -247,10 +248,6 @@ impl FrontendEnv {
         );
         info!("> version: {} ({})", RW_VERSION, GIT_SHA);
 
-        let batch_config = config.batch;
-        let meta_config = config.meta;
-        let streaming_config = config.streaming;
-
         let frontend_address: HostAddr = opts
             .advertise_addr
             .as_ref()
@@ -268,7 +265,7 @@ impl FrontendEnv {
             WorkerType::Frontend,
             &frontend_address,
             Default::default(),
-            &meta_config,
+            &config.meta,
         )
         .await?;
 
@@ -296,15 +293,16 @@ impl FrontendEnv {
         let frontend_meta_client = Arc::new(FrontendMetaClientImpl(meta_client.clone()));
         let hummock_snapshot_manager =
             Arc::new(HummockSnapshotManager::new(frontend_meta_client.clone()));
-        let compute_client_pool =
-            Arc::new(ComputeClientPool::new(config.server.connection_pool_size));
+        let compute_client_pool = Arc::new(ComputeClientPool::new(
+            config.batch_exchange_connection_pool_size(),
+        ));
         let query_manager = QueryManager::new(
             worker_node_manager.clone(),
             compute_client_pool.clone(),
             catalog_reader.clone(),
             Arc::new(GLOBAL_DISTRIBUTED_QUERY_METRICS.clone()),
-            batch_config.distributed_query_limit,
-            batch_config.max_batch_queries_per_frontend_node,
+            config.batch.distributed_query_limit,
+            config.batch.max_batch_queries_per_frontend_node,
         );
 
         let user_info_manager = Arc::new(RwLock::new(UserInfoManager::default()));
@@ -318,6 +316,12 @@ impl FrontendEnv {
         let system_params_manager =
             Arc::new(LocalSystemParamsManager::new(system_params_reader.clone()));
 
+        LocalSecretManager::init(
+            opts.temp_secret_file_dir,
+            meta_client.cluster_id().to_string(),
+            worker_id,
+        );
+
         // This `session_params` should be initialized during the initial notification in `observer_manager`
         let session_params = Arc::new(RwLock::new(SessionConfig::default()));
         let frontend_observer_node = FrontendObserverNode::new(
@@ -329,7 +333,7 @@ impl FrontendEnv {
             hummock_snapshot_manager.clone(),
             system_params_manager.clone(),
             session_params.clone(),
-            compute_client_pool,
+            compute_client_pool.clone(),
         );
         let observer_manager =
             ObserverManager::new_with_meta_client(meta_client.clone(), frontend_observer_node)
@@ -339,8 +343,6 @@ impl FrontendEnv {
 
         meta_client.activate(&frontend_address).await?;
 
-        let client_pool = Arc::new(ComputeClientPool::new(config.server.connection_pool_size));
-
         let frontend_metrics = Arc::new(GLOBAL_FRONTEND_METRICS.clone());
         let source_metrics = Arc::new(GLOBAL_SOURCE_METRICS.clone());
         let spill_metrics = Arc::new(GLOBAL_BATCH_SPILL_METRICS.clone());
@@ -384,7 +386,7 @@ impl FrontendEnv {
 
         let compute_runtime = Arc::new(BackgroundShutdownRuntime::from(
             Builder::new_multi_thread()
-                .worker_threads(batch_config.frontend_compute_runtime_worker_threads)
+                .worker_threads(config.batch.frontend_compute_runtime_worker_threads)
                 .thread_name("rw-batch-local")
                 .enable_all()
                 .build()
@@ -411,9 +413,11 @@ impl FrontendEnv {
 
         // Clean up the spill directory.
         #[cfg(not(madsim))]
-        SpillOp::clean_spill_directory()
-            .await
-            .map_err(|err| anyhow!(err))?;
+        if config.batch.enable_spill {
+            SpillOp::clean_spill_directory()
+                .await
+                .map_err(|err| anyhow!(err))?;
+        }
 
         let total_memory_bytes = resource_util::memory::system_memory_available_bytes();
         let heap_profiler =
@@ -439,13 +443,13 @@ impl FrontendEnv {
                 system_params_manager,
                 session_params,
                 server_addr: frontend_address,
-                client_pool,
+                client_pool: compute_client_pool,
                 frontend_metrics,
                 spill_metrics,
                 sessions_map,
-                batch_config,
-                meta_config,
-                streaming_config,
+                batch_config: config.batch,
+                meta_config: config.meta,
+                streaming_config: config.streaming,
                 source_metrics,
                 creating_streaming_job_tracker,
                 compute_runtime,
@@ -922,6 +926,27 @@ impl SessionImpl {
         Ok(connection.clone())
     }
 
+    pub fn get_subscription_by_schema_id_name(
+        &self,
+        schema_id: SchemaId,
+        subscription_name: &str,
+    ) -> Result<Arc<SubscriptionCatalog>> {
+        let db_name = self.database();
+
+        let catalog_reader = self.env().catalog_reader().read_guard();
+        let db_id = catalog_reader.get_database_by_name(db_name)?.id();
+        let schema = catalog_reader.get_schema_by_id(&db_id, &schema_id)?;
+        let subscription = schema
+            .get_subscription_by_name(subscription_name)
+            .ok_or_else(|| {
+                RwError::from(ErrorCode::ItemNotFound(format!(
+                    "subscription {} not found",
+                    subscription_name
+                )))
+            })?;
+        Ok(subscription.clone())
+    }
+
     pub fn get_subscription_by_name(
         &self,
         schema_name: Option<String>,
@@ -972,16 +997,41 @@ impl SessionImpl {
         Ok(table.clone())
     }
 
+    pub fn get_secret_by_name(
+        &self,
+        schema_name: Option<String>,
+        secret_name: &str,
+    ) -> Result<Arc<SecretCatalog>> {
+        let db_name = self.database();
+        let search_path = self.config().search_path();
+        let user_name = &self.auth_context().user_name;
+
+        let catalog_reader = self.env().catalog_reader().read_guard();
+        let schema = match schema_name {
+            Some(schema_name) => catalog_reader.get_schema_by_name(db_name, &schema_name)?,
+            None => catalog_reader.first_valid_schema(db_name, &search_path, user_name)?,
+        };
+        let schema = catalog_reader.get_schema_by_name(db_name, schema.name().as_str())?;
+        let secret = schema.get_secret_by_name(secret_name).ok_or_else(|| {
+            RwError::from(ErrorCode::ItemNotFound(format!(
+                "secret {} not found",
+                secret_name
+            )))
+        })?;
+        Ok(secret.clone())
+    }
+
     pub async fn list_change_log_epochs(
         &self,
         table_id: u32,
         min_epoch: u64,
         max_count: u32,
     ) -> Result<Vec<u64>> {
-        self.env
-            .catalog_writer
+        Ok(self
+            .env
+            .meta_client()
             .list_change_log_epochs(table_id, min_epoch, max_count)
-            .await
+            .await?)
     }
 
     pub fn clear_cancel_query_flag(&self) {
diff --git a/src/frontend/src/session/cursor_manager.rs b/src/frontend/src/session/cursor_manager.rs
index 46eca3beb9966..bcd1aa11ec749 100644
--- a/src/frontend/src/session/cursor_manager.rs
+++ b/src/frontend/src/session/cursor_manager.rs
@@ -214,9 +214,10 @@ impl SubscriptionCursor {
                     // Initiate a new batch query to continue fetching
                     match Self::get_next_rw_timestamp(
                         *seek_timestamp,
-                        self.dependent_table_id.table_id,
+                        &self.dependent_table_id,
                         *expected_timestamp,
                         handle_args.clone(),
+                        &self.subscription,
                     )
                     .await
                     {
@@ -343,15 +344,21 @@ impl SubscriptionCursor {
 
     async fn get_next_rw_timestamp(
         seek_timestamp: u64,
-        table_id: u32,
+        table_id: &TableId,
         expected_timestamp: Option<u64>,
         handle_args: HandlerArgs,
+        dependent_subscription: &SubscriptionCatalog,
     ) -> Result<(Option<u64>, Option<u64>)> {
+        let session = handle_args.session;
+        // Test subscription existence
+        session.get_subscription_by_schema_id_name(
+            dependent_subscription.schema_id,
+            &dependent_subscription.name,
+        )?;
+
         // The epoch here must be pulled every time, otherwise there will be cache consistency issues
-        let new_epochs = handle_args
-            .session
-            .catalog_writer()?
-            .list_change_log_epochs(table_id, seek_timestamp, 2)
+        let new_epochs = session
+            .list_change_log_epochs(table_id.table_id(), seek_timestamp, 2)
             .await?;
         if let Some(expected_timestamp) = expected_timestamp
             && (new_epochs.is_empty() || &expected_timestamp != new_epochs.first().unwrap())
@@ -525,9 +532,17 @@ impl CursorManager {
             handle_args,
         )
         .await?;
-        self.cursor_map
-            .lock()
-            .await
+        let mut cursor_map = self.cursor_map.lock().await;
+
+        cursor_map.retain(|_, v| {
+            if let Cursor::Subscription(cursor) = v {
+                !matches!(cursor.state, State::Invalid)
+            } else {
+                true
+            }
+        });
+
+        cursor_map
             .try_insert(cursor.cursor_name.clone(), Cursor::Subscription(cursor))
             .map_err(|_| {
                 ErrorCode::CatalogError(format!("cursor `{}` already exists", cursor_name).into())
diff --git a/src/frontend/src/stream_fragmenter/mod.rs b/src/frontend/src/stream_fragmenter/mod.rs
index f2d768cc076f4..7596a5544b391 100644
--- a/src/frontend/src/stream_fragmenter/mod.rs
+++ b/src/frontend/src/stream_fragmenter/mod.rs
@@ -308,7 +308,7 @@ fn build_fragment(
                 // memorize upstream source id for later use
                 state
                     .dependent_table_ids
-                    .insert(TableId::new(node.upstream_source_id));
+                    .insert(node.upstream_source_id.into());
                 current_fragment
                     .upstream_table_ids
                     .push(node.upstream_source_id);
diff --git a/src/frontend/src/telemetry.rs b/src/frontend/src/telemetry.rs
index d50b9999b946f..19777a28ca901 100644
--- a/src/frontend/src/telemetry.rs
+++ b/src/frontend/src/telemetry.rs
@@ -14,14 +14,35 @@
 
 use prost::Message;
 use risingwave_common::telemetry::pb_compatible::TelemetryToProtobuf;
-use risingwave_common::telemetry::report::TelemetryReportCreator;
+use risingwave_common::telemetry::report::{report_event_common, TelemetryReportCreator};
 use risingwave_common::telemetry::{
     current_timestamp, SystemData, TelemetryNodeType, TelemetryReportBase, TelemetryResult,
 };
+use risingwave_pb::telemetry::{PbTelemetryDatabaseObject, PbTelemetryEventStage};
 use serde::{Deserialize, Serialize};
 
 const TELEMETRY_FRONTEND_REPORT_TYPE: &str = "frontend";
 
+#[allow(dead_code)] // please remove when used
+pub(crate) fn report_event(
+    event_stage: PbTelemetryEventStage,
+    event_name: &str,
+    catalog_id: i64,
+    connector_name: Option<String>,
+    component: Option<PbTelemetryDatabaseObject>,
+    attributes: Option<jsonbb::Value>, // any json string
+) {
+    report_event_common(
+        event_stage,
+        event_name,
+        catalog_id,
+        connector_name,
+        component,
+        attributes,
+        TELEMETRY_FRONTEND_REPORT_TYPE.to_string(),
+    );
+}
+
 #[derive(Clone, Copy)]
 pub(crate) struct FrontendTelemetryCreator {}
 
diff --git a/src/frontend/src/test_utils.rs b/src/frontend/src/test_utils.rs
index a9ff4084df443..24453f766c72c 100644
--- a/src/frontend/src/test_utils.rs
+++ b/src/frontend/src/test_utils.rs
@@ -42,7 +42,7 @@ use risingwave_pb::common::WorkerNode;
 use risingwave_pb::ddl_service::alter_owner_request::Object;
 use risingwave_pb::ddl_service::{
     alter_set_schema_request, create_connection_request, DdlProgress, PbTableJobType,
-    ReplaceTablePlan,
+    ReplaceTablePlan, TableJobType,
 };
 use risingwave_pb::hummock::write_limits::WriteLimit;
 use risingwave_pb::hummock::{
@@ -55,7 +55,9 @@ use risingwave_pb::meta::list_fragment_distribution_response::FragmentDistributi
 use risingwave_pb::meta::list_object_dependencies_response::PbObjectDependencies;
 use risingwave_pb::meta::list_table_fragment_states_response::TableFragmentState;
 use risingwave_pb::meta::list_table_fragments_response::TableFragmentInfo;
-use risingwave_pb::meta::{EventLog, PbTableParallelism, PbThrottleTarget, SystemParams};
+use risingwave_pb::meta::{
+    EventLog, PbTableParallelism, PbThrottleTarget, RecoveryStatus, SystemParams,
+};
 use risingwave_pb::stream_plan::StreamFragmentGraph;
 use risingwave_pb::user::update_user_request::UpdateField;
 use risingwave_pb::user::{GrantPrivilege, UserInfo};
@@ -242,15 +244,6 @@ impl CatalogWriter for MockCatalogWriter {
         Ok(())
     }
 
-    async fn list_change_log_epochs(
-        &self,
-        _table_id: u32,
-        _min_epoch: u64,
-        _max_count: u32,
-    ) -> Result<Vec<u64>> {
-        unreachable!()
-    }
-
     async fn create_schema(
         &self,
         db_id: DatabaseId,
@@ -309,6 +302,7 @@ impl CatalogWriter for MockCatalogWriter {
         mut table: PbTable,
         _graph: StreamFragmentGraph,
         _mapping: ColIndexMapping,
+        _job_type: TableJobType,
     ) -> Result<()> {
         table.stream_job_status = PbStreamJobStatus::Created as _;
         self.catalog.write().update_table(&table);
@@ -1091,6 +1085,19 @@ impl FrontendMetaClient for MockFrontendMetaClient {
     ) -> RpcResult<()> {
         unimplemented!()
     }
+
+    async fn get_cluster_recovery_status(&self) -> RpcResult<RecoveryStatus> {
+        Ok(RecoveryStatus::StatusRunning)
+    }
+
+    async fn list_change_log_epochs(
+        &self,
+        _table_id: u32,
+        _min_epoch: u64,
+        _max_count: u32,
+    ) -> RpcResult<Vec<u64>> {
+        unimplemented!()
+    }
 }
 
 #[cfg(test)]
diff --git a/src/frontend/src/user/user_authentication.rs b/src/frontend/src/user/user_authentication.rs
index b0cefc1faedcb..a36c09e987a7a 100644
--- a/src/frontend/src/user/user_authentication.rs
+++ b/src/frontend/src/user/user_authentication.rs
@@ -35,9 +35,8 @@ pub const OAUTH_ISSUER_KEY: &str = "issuer";
 /// Build `AuthInfo` for `OAuth`.
 #[inline(always)]
 pub fn build_oauth_info(options: &Vec<SqlOption>) -> Option<AuthInfo> {
-    let metadata: HashMap<String, String> = WithOptions::try_from(options.as_slice())
+    let metadata: HashMap<String, String> = WithOptions::oauth_options_to_map(options.as_slice())
         .ok()?
-        .into_inner()
         .into_iter()
         .collect();
     if !metadata.contains_key(OAUTH_JWKS_URL_KEY) || !metadata.contains_key(OAUTH_ISSUER_KEY) {
diff --git a/src/frontend/src/utils/column_index_mapping.rs b/src/frontend/src/utils/column_index_mapping.rs
index 08343eb9f09ce..4a7a729eb9b72 100644
--- a/src/frontend/src/utils/column_index_mapping.rs
+++ b/src/frontend/src/utils/column_index_mapping.rs
@@ -20,7 +20,8 @@ use risingwave_common::util::sort_util::ColumnOrder;
 
 use crate::expr::{Expr, ExprImpl, ExprRewriter, InputRef};
 use crate::optimizer::property::{
-    Distribution, FunctionalDependency, FunctionalDependencySet, Order, RequiredDist,
+    Distribution, FunctionalDependency, FunctionalDependencySet, MonotonicityMap, Order,
+    RequiredDist,
 };
 
 /// Extension trait for [`ColIndexMapping`] to rewrite frontend structures.
@@ -186,6 +187,16 @@ impl ColIndexMapping {
         }
         ret
     }
+
+    pub fn rewrite_monotonicity_map(&self, map: &MonotonicityMap) -> MonotonicityMap {
+        let mut new_map = MonotonicityMap::new();
+        for (i, monotonicity) in map.iter() {
+            if let Some(mapped_i) = self.try_map(i) {
+                new_map.insert(mapped_i, monotonicity);
+            }
+        }
+        new_map
+    }
 }
 
 impl ExprRewriter for ColIndexMapping {
diff --git a/src/frontend/src/utils/overwrite_options.rs b/src/frontend/src/utils/overwrite_options.rs
index e1719c348107f..185f1b80b154b 100644
--- a/src/frontend/src/utils/overwrite_options.rs
+++ b/src/frontend/src/utils/overwrite_options.rs
@@ -16,24 +16,34 @@ use crate::handler::HandlerArgs;
 
 #[derive(Debug, Clone, Default)]
 pub struct OverwriteOptions {
-    pub streaming_rate_limit: Option<u32>,
+    pub source_rate_limit: Option<u32>,
+    pub backfill_rate_limit: Option<u32>,
 }
 
 impl OverwriteOptions {
-    pub(crate) const STREAMING_RATE_LIMIT_KEY: &'static str = "streaming_rate_limit";
+    pub(crate) const BACKFILL_RATE_LIMIT_KEY: &'static str = "backfill_rate_limit";
+    pub(crate) const SOURCE_RATE_LIMIT_KEY: &'static str = "source_rate_limit";
 
     pub fn new(args: &mut HandlerArgs) -> Self {
-        let streaming_rate_limit = {
-            // CREATE MATERIALIZED VIEW m1 WITH (rate_limit = N) ...
-            if let Some(x) = args
-                .with_options
-                .inner_mut()
-                .remove(Self::STREAMING_RATE_LIMIT_KEY)
-            {
+        let source_rate_limit = {
+            if let Some(x) = args.with_options.remove(Self::SOURCE_RATE_LIMIT_KEY) {
                 // FIXME(tabVersion): validate the value
                 Some(x.parse::<u32>().unwrap())
             } else {
-                let rate_limit = args.session.config().streaming_rate_limit();
+                let rate_limit = args.session.config().source_rate_limit();
+                if rate_limit < 0 {
+                    None
+                } else {
+                    Some(rate_limit as u32)
+                }
+            }
+        };
+        let backfill_rate_limit = {
+            if let Some(x) = args.with_options.remove(Self::BACKFILL_RATE_LIMIT_KEY) {
+                // FIXME(tabVersion): validate the value
+                Some(x.parse::<u32>().unwrap())
+            } else {
+                let rate_limit = args.session.config().backfill_rate_limit();
                 if rate_limit < 0 {
                     None
                 } else {
@@ -42,7 +52,8 @@ impl OverwriteOptions {
             }
         };
         Self {
-            streaming_rate_limit,
+            source_rate_limit,
+            backfill_rate_limit,
         }
     }
 }
diff --git a/src/frontend/src/utils/with_options.rs b/src/frontend/src/utils/with_options.rs
index a6984d687bc74..e306103c02e39 100644
--- a/src/frontend/src/utils/with_options.rs
+++ b/src/frontend/src/utils/with_options.rs
@@ -18,11 +18,13 @@ use std::num::NonZeroU32;
 use risingwave_connector::source::kafka::private_link::{
     insert_privatelink_broker_rewrite_map, CONNECTION_NAME_KEY, PRIVATELINK_ENDPOINT_KEY,
 };
+pub use risingwave_connector::WithOptionsSecResolved;
 use risingwave_connector::WithPropertiesExt;
+use risingwave_pb::secret::secret_ref::PbRefAsType;
 use risingwave_pb::secret::PbSecretRef;
 use risingwave_sqlparser::ast::{
     CreateConnectionStatement, CreateSinkStatement, CreateSourceStatement,
-    CreateSubscriptionStatement, SqlOption, Statement, Value,
+    CreateSubscriptionStatement, SecretRef, SecretRefAsType, SqlOption, Statement, Value,
 };
 
 use super::OverwriteOptions;
@@ -30,16 +32,18 @@ use crate::catalog::connection_catalog::resolve_private_link_connection;
 use crate::catalog::ConnectionId;
 use crate::error::{ErrorCode, Result as RwResult, RwError};
 use crate::session::SessionImpl;
+use crate::Binder;
 
 mod options {
 
     pub const RETENTION_SECONDS: &str = "retention_seconds";
 }
 
-/// Options or properties extracted fro m the `WITH` clause of DDLs.
+/// Options or properties extracted from the `WITH` clause of DDLs.
 #[derive(Default, Clone, Debug, PartialEq, Eq, Hash)]
 pub struct WithOptions {
     inner: BTreeMap<String, String>,
+    secret_ref: BTreeMap<String, SecretRef>,
 }
 
 impl std::ops::Deref for WithOptions {
@@ -58,35 +62,41 @@ impl std::ops::DerefMut for WithOptions {
 
 impl WithOptions {
     /// Create a new [`WithOptions`] from a [`BTreeMap`].
-    pub fn new(inner: BTreeMap<String, String>) -> Self {
+    pub fn new_with_options(inner: BTreeMap<String, String>) -> Self {
         Self {
-            inner: inner.into_iter().collect(),
+            inner,
+            secret_ref: Default::default(),
         }
     }
 
-    /// Get the reference of the inner map.
-    pub fn inner(&self) -> &BTreeMap<String, String> {
-        &self.inner
+    /// Create a new [`WithOptions`] from a option [`BTreeMap`] and secret ref.
+    pub fn new(inner: BTreeMap<String, String>, secret_ref: BTreeMap<String, SecretRef>) -> Self {
+        Self { inner, secret_ref }
     }
 
     pub fn inner_mut(&mut self) -> &mut BTreeMap<String, String> {
         &mut self.inner
     }
 
-    /// Take the value of the inner map.
-    pub fn into_inner(self) -> BTreeMap<String, String> {
-        self.inner
+    /// Take the value of the option map and secret refs.
+    pub fn into_parts(self) -> (BTreeMap<String, String>, BTreeMap<String, SecretRef>) {
+        (self.inner, self.secret_ref)
     }
 
     /// Convert to connector props, remove the key-value pairs used in the top-level.
-    pub fn into_connector_props(self) -> BTreeMap<String, String> {
-        self.inner
+    pub fn into_connector_props(self) -> WithOptions {
+        let inner = self
+            .inner
             .into_iter()
             .filter(|(key, _)| {
-                key != OverwriteOptions::STREAMING_RATE_LIMIT_KEY
-                    && key != options::RETENTION_SECONDS
+                key != OverwriteOptions::SOURCE_RATE_LIMIT_KEY && key != options::RETENTION_SECONDS
             })
-            .collect()
+            .collect();
+
+        Self {
+            inner,
+            secret_ref: self.secret_ref,
+        }
     }
 
     /// Parse the retention seconds from the options.
@@ -107,7 +117,10 @@ impl WithOptions {
             })
             .collect();
 
-        Self { inner }
+        Self {
+            inner,
+            secret_ref: self.secret_ref.clone(),
+        }
     }
 
     pub fn value_eq_ignore_case(&self, key: &str, val: &str) -> bool {
@@ -118,15 +131,57 @@ impl WithOptions {
         }
         false
     }
-}
 
-pub(crate) fn resolve_secret_in_with_options(
-    _with_options: &mut WithOptions,
-    _session: &SessionImpl,
-) -> RwResult<BTreeMap<String, PbSecretRef>> {
-    // todo: implement the function and take `resolve_privatelink_in_with_option` as reference
+    pub fn secret_ref(&self) -> &BTreeMap<String, SecretRef> {
+        &self.secret_ref
+    }
+
+    pub fn encode_options_to_map(sql_options: &[SqlOption]) -> RwResult<BTreeMap<String, String>> {
+        let WithOptions { inner, secret_ref } = WithOptions::try_from(sql_options)?;
+        if secret_ref.is_empty() {
+            Ok(inner)
+        } else {
+            Err(RwError::from(ErrorCode::InvalidParameterValue(
+                "Secret reference is not allowed in encode options".to_string(),
+            )))
+        }
+    }
 
-    Ok(BTreeMap::new())
+    pub fn oauth_options_to_map(sql_options: &[SqlOption]) -> RwResult<BTreeMap<String, String>> {
+        let WithOptions { inner, secret_ref } = WithOptions::try_from(sql_options)?;
+        if secret_ref.is_empty() {
+            Ok(inner)
+        } else {
+            Err(RwError::from(ErrorCode::InvalidParameterValue(
+                "Secret reference is not allowed in OAuth options".to_string(),
+            )))
+        }
+    }
+}
+
+/// Get the secret id from the name.
+pub(crate) fn resolve_secret_ref_in_with_options(
+    with_options: WithOptions,
+    session: &SessionImpl,
+) -> RwResult<WithOptionsSecResolved> {
+    let (options, secret_refs) = with_options.into_parts();
+    let mut resolved_secret_refs = BTreeMap::new();
+    let db_name: &str = session.database();
+    for (key, secret_ref) in secret_refs {
+        let (schema_name, secret_name) =
+            Binder::resolve_schema_qualified_name(db_name, secret_ref.secret_name.clone())?;
+        let secret_catalog = session.get_secret_by_name(schema_name, &secret_name)?;
+        let ref_as = match secret_ref.ref_as {
+            SecretRefAsType::Text => PbRefAsType::Text,
+            SecretRefAsType::File => PbRefAsType::File,
+        };
+        let pb_secret_ref = PbSecretRef {
+            secret_id: secret_catalog.id.secret_id(),
+            ref_as: ref_as.into(),
+        };
+        resolved_secret_refs.insert(key.clone(), pb_secret_ref);
+    }
+    Ok(WithOptionsSecResolved::new(options, resolved_secret_refs))
 }
 
 pub(crate) fn resolve_privatelink_in_with_option(
@@ -175,8 +230,18 @@ impl TryFrom<&[SqlOption]> for WithOptions {
 
     fn try_from(options: &[SqlOption]) -> Result<Self, Self::Error> {
         let mut inner: BTreeMap<String, String> = BTreeMap::new();
+        let mut secret_ref: BTreeMap<String, SecretRef> = BTreeMap::new();
         for option in options {
             let key = option.name.real_value();
+            if let Value::Ref(r) = &option.value {
+                if secret_ref.insert(key.clone(), r.clone()).is_some() || inner.contains_key(&key) {
+                    return Err(RwError::from(ErrorCode::InvalidParameterValue(format!(
+                        "Duplicated option: {}",
+                        key
+                    ))));
+                }
+                continue;
+            }
             let value: String = match option.value.clone() {
                 Value::CstyleEscapedString(s) => s.value,
                 Value::SingleQuotedString(s) => s,
@@ -189,7 +254,7 @@ impl TryFrom<&[SqlOption]> for WithOptions {
                     )))
                 }
             };
-            if inner.insert(key.clone(), value).is_some() {
+            if inner.insert(key.clone(), value).is_some() || secret_ref.contains_key(&key) {
                 return Err(RwError::from(ErrorCode::InvalidParameterValue(format!(
                     "Duplicated option: {}",
                     key
@@ -197,7 +262,7 @@ impl TryFrom<&[SqlOption]> for WithOptions {
             }
         }
 
-        Ok(Self { inner })
+        Ok(Self { inner, secret_ref })
     }
 }
 
diff --git a/src/jni_core/src/lib.rs b/src/jni_core/src/lib.rs
index 18d1807948d21..419f4ffd21cb5 100644
--- a/src/jni_core/src/lib.rs
+++ b/src/jni_core/src/lib.rs
@@ -13,7 +13,6 @@
 // limitations under the License.
 
 #![feature(error_generic_member_access)]
-#![feature(lazy_cell)]
 #![feature(once_cell_try)]
 #![feature(type_alias_impl_trait)]
 #![feature(try_blocks)]
@@ -243,25 +242,28 @@ struct JavaClassMethodCache {
     utc: OnceLock<GlobalRef>,
 }
 
-// TODO: may only return a RowRef
-pub type StreamChunkRowIterator<'a> = impl Iterator<Item = (Op, OwnedRow)> + 'a;
+mod opaque_type {
+    use super::*;
+    // TODO: may only return a RowRef
+    pub type StreamChunkRowIterator<'a> = impl Iterator<Item = (Op, OwnedRow)> + 'a;
+
+    impl<'a> JavaBindingIteratorInner<'a> {
+        pub(super) fn from_chunk(chunk: &'a StreamChunk) -> JavaBindingIteratorInner<'a> {
+            JavaBindingIteratorInner::StreamChunk(
+                chunk
+                    .rows()
+                    .map(|(op, row)| (op.to_protobuf(), row.to_owned_row())),
+            )
+        }
+    }
+}
+pub use opaque_type::StreamChunkRowIterator;
 pub type HummockJavaBindingIterator = BoxStream<'static, anyhow::Result<(Bytes, OwnedRow)>>;
-
 pub enum JavaBindingIteratorInner<'a> {
     Hummock(HummockJavaBindingIterator),
     StreamChunk(StreamChunkRowIterator<'a>),
 }
 
-impl<'a> JavaBindingIteratorInner<'a> {
-    fn from_chunk(chunk: &'a StreamChunk) -> JavaBindingIteratorInner<'a> {
-        JavaBindingIteratorInner::StreamChunk(
-            chunk
-                .rows()
-                .map(|(op, row)| (op.to_protobuf(), row.to_owned_row())),
-        )
-    }
-}
-
 enum RowExtra {
     Op(Op),
     Key(Bytes),
diff --git a/src/jni_core/src/macros.rs b/src/jni_core/src/macros.rs
index 982ccda06ecf0..2e7d095e0bd45 100644
--- a/src/jni_core/src/macros.rs
+++ b/src/jni_core/src/macros.rs
@@ -375,7 +375,6 @@ macro_rules! to_jvalue {
 
 /// Generate the jni signature of a given function
 /// ```
-/// #![feature(lazy_cell)]
 /// use risingwave_jni_core::gen_jni_sig;
 /// assert_eq!(gen_jni_sig!(boolean f(int, short, byte[])), "(IS[B)Z");
 /// assert_eq!(
diff --git a/src/license/src/feature.rs b/src/license/src/feature.rs
index 302538cc3ecc3..00c6d200aa0a3 100644
--- a/src/license/src/feature.rs
+++ b/src/license/src/feature.rs
@@ -44,6 +44,13 @@ macro_rules! for_all_features {
         $macro! {
             // name                 min tier    doc
             { TestPaid,             Paid,       "A dummy feature that's only available on paid tier for testing purposes." },
+            { TimeTravel,           Paid,       "Query historical data within the retention period."},
+            { GlueSchemaRegistry,   Paid,       "Use Schema Registry from AWS Glue rather than Confluent." },
+            { ClickHouseSharedEngine,Paid,      "Delivering data to Shared tree on clickhouse cloud"},
+            { SecretManagement,     Paid,       "Secret management." },
+            { CdcTableSchemaMap,    Paid,       "Automatically map upstream schema to CDC Table."},
+            { SqlServerSink,        Paid,       "Sink data from RisingWave to SQL Server." },
+            { SqlServerCdcSource,   Paid,       "CDC source connector for Sql Server." },
         }
     };
 }
@@ -80,9 +87,9 @@ for_all_features!(def_feature);
 #[derive(Debug, Error)]
 pub enum FeatureNotAvailable {
     #[error(
-        "feature {:?} is only available for tier {:?} and above, while the current tier is {:?}\n\n\
+    "feature {:?} is only available for tier {:?} and above, while the current tier is {:?}\n\n\
         Hint: You may want to set a license key with `ALTER SYSTEM SET license_key = '...';` command.",
-        feature, feature.min_tier(), current_tier,
+    feature, feature.min_tier(), current_tier,
     )]
     InsufficientTier {
         feature: Feature,
diff --git a/src/license/src/lib.rs b/src/license/src/lib.rs
index 0e641be9789b1..7f2b25d8f3fb4 100644
--- a/src/license/src/lib.rs
+++ b/src/license/src/lib.rs
@@ -12,8 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#![feature(lazy_cell)]
-
 mod feature;
 mod manager;
 
diff --git a/src/meta/Cargo.toml b/src/meta/Cargo.toml
index ddae0c9c24626..9e35e94896e5a 100644
--- a/src/meta/Cargo.toml
+++ b/src/meta/Cargo.toml
@@ -14,7 +14,6 @@ ignored = ["workspace-hack"]
 normal = ["workspace-hack"]
 
 [dependencies]
-aes-siv = "0.7"
 anyhow = "1"
 arc-swap = "1"
 assert_matches = "1"
@@ -39,6 +38,7 @@ futures = { version = "0.3", default-features = false, features = ["alloc"] }
 hex = "0.4"
 hyper = "0.14" # required by tonic
 itertools = { workspace = true }
+jsonbb = { workspace = true }
 maplit = "1.0.2"
 memcomparable = { version = "0.2" }
 mime_guess = "2"
@@ -72,12 +72,12 @@ sync-point = { path = "../utils/sync-point" }
 thiserror = "1"
 thiserror-ext = { workspace = true }
 tokio = { version = "0.2", package = "madsim-tokio", features = [
-    "rt",
-    "rt-multi-thread",
-    "sync",
-    "macros",
-    "time",
-    "signal",
+  "rt",
+  "rt-multi-thread",
+  "sync",
+  "macros",
+  "time",
+  "signal",
 ] }
 tokio-retry = "0.3"
 tokio-stream = { version = "0.1", features = ["net"] }
@@ -90,10 +90,10 @@ uuid = { version = "1", features = ["v4"] }
 [target.'cfg(not(madsim))'.dependencies]
 axum = { workspace = true }
 tower-http = { version = "0.5", features = [
-    "add-extension",
-    "cors",
-    "fs",
-    "compression-gzip",
+  "add-extension",
+  "cors",
+  "fs",
+  "compression-gzip",
 ] }
 workspace-hack = { path = "../workspace-hack" }
 
diff --git a/src/meta/model_v2/migration/Cargo.toml b/src/meta/model_v2/migration/Cargo.toml
index ac9cd1b0d157b..a54753da16fbb 100644
--- a/src/meta/model_v2/migration/Cargo.toml
+++ b/src/meta/model_v2/migration/Cargo.toml
@@ -15,6 +15,7 @@ normal = ["workspace-hack"]
 
 [dependencies]
 async-std = { version = "1", features = ["attributes", "tokio1"] }
+sea-orm = { workspace = true }
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 uuid = { version = "1", features = ["v4"] }
diff --git a/src/meta/model_v2/migration/README.md b/src/meta/model_v2/migration/README.md
index d68a624ced8a7..2ed136b8d60f1 100644
--- a/src/meta/model_v2/migration/README.md
+++ b/src/meta/model_v2/migration/README.md
@@ -9,7 +9,9 @@
 > **DO NOT** modify already published migration files.
 
 ## How to run the migrator CLI
+
 - Generate a new migration file, a database endpoint is required but not used.
+  Run this command in this directory, not project root.
     ```sh
     export DATABASE_URL=sqlite::memory:; cargo run -- generate MIGRATION_NAME
     ```
@@ -48,3 +50,7 @@
     ```sh
     cargo run -- status
     ```
+
+## Adding a migration
+
+- Add a new column to some catalogs. You can checkout the migration [m20240617_070131_index_column_properties.rs](src/m20240617_070131_index_column_properties.rs) as a reference.
\ No newline at end of file
diff --git a/src/meta/model_v2/migration/src/lib.rs b/src/meta/model_v2/migration/src/lib.rs
index a835bf0b9e6e8..dc1f3dd3f9808 100644
--- a/src/meta/model_v2/migration/src/lib.rs
+++ b/src/meta/model_v2/migration/src/lib.rs
@@ -12,9 +12,13 @@ mod m20240418_142249_function_runtime;
 mod m20240506_112555_subscription_partial_ckpt;
 mod m20240525_090457_secret;
 mod m20240617_070131_index_column_properties;
+mod m20240617_071625_sink_into_table_column;
 mod m20240618_072634_function_compressed_binary;
+mod m20240630_131430_remove_parallel_unit;
+mod m20240701_060504_hummock_time_travel;
 mod m20240702_080451_system_param_value;
 mod m20240702_084927_unnecessary_fk;
+mod m20240806_143329_add_rate_limit_to_source_catalog;
 
 pub struct Migrator;
 
@@ -31,10 +35,14 @@ impl MigratorTrait for Migrator {
             Box::new(m20240418_142249_function_runtime::Migration),
             Box::new(m20240506_112555_subscription_partial_ckpt::Migration),
             Box::new(m20240525_090457_secret::Migration),
-            Box::new(m20240618_072634_function_compressed_binary::Migration),
             Box::new(m20240617_070131_index_column_properties::Migration),
+            Box::new(m20240617_071625_sink_into_table_column::Migration),
+            Box::new(m20240618_072634_function_compressed_binary::Migration),
+            Box::new(m20240630_131430_remove_parallel_unit::Migration),
+            Box::new(m20240701_060504_hummock_time_travel::Migration),
             Box::new(m20240702_080451_system_param_value::Migration),
             Box::new(m20240702_084927_unnecessary_fk::Migration),
+            Box::new(m20240806_143329_add_rate_limit_to_source_catalog::Migration),
         ]
     }
 }
diff --git a/src/meta/model_v2/migration/src/m20240617_071625_sink_into_table_column.rs b/src/meta/model_v2/migration/src/m20240617_071625_sink_into_table_column.rs
new file mode 100644
index 0000000000000..4904dd3623245
--- /dev/null
+++ b/src/meta/model_v2/migration/src/m20240617_071625_sink_into_table_column.rs
@@ -0,0 +1,70 @@
+use sea_orm_migration::prelude::{Table as MigrationTable, *};
+
+use crate::SubQueryStatement::SelectStatement;
+
+#[derive(DeriveMigrationName)]
+pub struct Migration;
+
+#[async_trait::async_trait]
+impl MigrationTrait for Migration {
+    async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
+        manager
+            .alter_table(
+                MigrationTable::alter()
+                    .table(Sink::Table)
+                    .add_column(ColumnDef::new(Sink::OriginalTargetColumns).binary())
+                    .to_owned(),
+            )
+            .await?;
+
+        let stmt = Query::update()
+            .table(Sink::Table)
+            .value(
+                Sink::OriginalTargetColumns,
+                SimpleExpr::SubQuery(
+                    None,
+                    Box::new(SelectStatement(
+                        Query::select()
+                            .column((Table::Table, Table::Columns))
+                            .from(Table::Table)
+                            .and_where(
+                                Expr::col((Table::Table, Table::TableId))
+                                    .equals((Sink::Table, Sink::TargetTable)),
+                            )
+                            .to_owned(),
+                    )),
+                ),
+            )
+            .and_where(Expr::col((Sink::Table, Sink::TargetTable)).is_not_null())
+            .to_owned();
+
+        manager.exec_stmt(stmt).await?;
+
+        Ok(())
+    }
+
+    async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> {
+        manager
+            .alter_table(
+                MigrationTable::alter()
+                    .table(Sink::Table)
+                    .drop_column(Sink::OriginalTargetColumns)
+                    .to_owned(),
+            )
+            .await
+    }
+}
+
+#[derive(DeriveIden)]
+enum Sink {
+    Table,
+    TargetTable,
+    OriginalTargetColumns,
+}
+
+#[derive(DeriveIden)]
+enum Table {
+    Table,
+    TableId,
+    Columns,
+}
diff --git a/src/meta/model_v2/migration/src/m20240630_131430_remove_parallel_unit.rs b/src/meta/model_v2/migration/src/m20240630_131430_remove_parallel_unit.rs
new file mode 100644
index 0000000000000..5acc8dc1a6165
--- /dev/null
+++ b/src/meta/model_v2/migration/src/m20240630_131430_remove_parallel_unit.rs
@@ -0,0 +1,160 @@
+use sea_orm_migration::prelude::*;
+use serde::{Deserialize, Serialize};
+
+use crate::sea_orm::{FromJsonQueryResult, FromQueryResult, Statement};
+
+#[derive(DeriveMigrationName)]
+pub struct Migration;
+
+#[async_trait::async_trait]
+impl MigrationTrait for Migration {
+    async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
+        manager
+            .alter_table(
+                Table::alter()
+                    .table(WorkerProperty::Table)
+                    .add_column(ColumnDef::new(WorkerProperty::Parallelism).integer())
+                    .to_owned(),
+            )
+            .await?;
+
+        set_worker_parallelism(manager).await?;
+
+        manager
+            .alter_table(
+                Table::alter()
+                    .table(WorkerProperty::Table)
+                    .drop_column(WorkerProperty::ParallelUnitIds)
+                    .to_owned(),
+            )
+            .await?;
+
+        manager
+            .alter_table(
+                Table::alter()
+                    .table(Fragment::Table)
+                    .drop_column(Fragment::VnodeMapping)
+                    .to_owned(),
+            )
+            .await?;
+
+        manager
+            .alter_table(
+                Table::alter()
+                    .table(Actor::Table)
+                    .drop_column(Actor::ParallelUnitId)
+                    .to_owned(),
+            )
+            .await
+    }
+
+    async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> {
+        manager
+            .alter_table(
+                Table::alter()
+                    .table(WorkerProperty::Table)
+                    .drop_column(WorkerProperty::Parallelism)
+                    .to_owned(),
+            )
+            .await?;
+
+        manager
+            .alter_table(
+                Table::alter()
+                    .table(WorkerProperty::Table)
+                    .add_column(
+                        ColumnDef::new(WorkerProperty::ParallelUnitIds)
+                            .json_binary()
+                            .not_null(),
+                    )
+                    .to_owned(),
+            )
+            .await?;
+
+        manager
+            .alter_table(
+                Table::alter()
+                    .table(Fragment::Table)
+                    .add_column(ColumnDef::new(Fragment::VnodeMapping).binary().not_null())
+                    .to_owned(),
+            )
+            .await?;
+
+        manager
+            .alter_table(
+                Table::alter()
+                    .table(Actor::Table)
+                    .add_column(ColumnDef::new(Actor::ParallelUnitId).integer().not_null())
+                    .to_owned(),
+            )
+            .await
+    }
+}
+
+// Set worker parallelism based on the number of parallel unit ids
+async fn set_worker_parallelism(manager: &SchemaManager<'_>) -> Result<(), DbErr> {
+    let connection = manager.get_connection();
+
+    let database_backend = connection.get_database_backend();
+
+    let (sql, values) = Query::select()
+        .columns([
+            (WorkerProperty::Table, WorkerProperty::WorkerId),
+            (WorkerProperty::Table, WorkerProperty::ParallelUnitIds),
+        ])
+        .from(WorkerProperty::Table)
+        .to_owned()
+        .build_any(&*database_backend.get_query_builder());
+
+    let stmt = Statement::from_sql_and_values(database_backend, sql, values);
+
+    for WorkerPropertyParallelUnitIds {
+        worker_id,
+        parallel_unit_ids,
+    } in WorkerPropertyParallelUnitIds::find_by_statement(stmt)
+        .all(connection)
+        .await?
+    {
+        manager
+            .exec_stmt(
+                Query::update()
+                    .table(WorkerProperty::Table)
+                    .value(
+                        WorkerProperty::Parallelism,
+                        Expr::value(parallel_unit_ids.0.len() as i32),
+                    )
+                    .and_where(Expr::col(WorkerProperty::WorkerId).eq(worker_id))
+                    .to_owned(),
+            )
+            .await?;
+    }
+    Ok(())
+}
+#[derive(Clone, Debug, PartialEq, FromJsonQueryResult, Serialize, Deserialize, Default)]
+pub struct I32Array(pub Vec<i32>);
+
+#[derive(Debug, FromQueryResult)]
+pub struct WorkerPropertyParallelUnitIds {
+    worker_id: i32,
+    parallel_unit_ids: I32Array,
+}
+
+#[derive(DeriveIden)]
+enum WorkerProperty {
+    Table,
+    WorkerId,
+    Parallelism,
+    ParallelUnitIds,
+}
+
+#[derive(DeriveIden)]
+enum Fragment {
+    Table,
+    VnodeMapping,
+}
+
+#[derive(DeriveIden)]
+enum Actor {
+    Table,
+    ParallelUnitId,
+}
diff --git a/src/meta/model_v2/migration/src/m20240701_060504_hummock_time_travel.rs b/src/meta/model_v2/migration/src/m20240701_060504_hummock_time_travel.rs
new file mode 100644
index 0000000000000..7dec44913dc8f
--- /dev/null
+++ b/src/meta/model_v2/migration/src/m20240701_060504_hummock_time_travel.rs
@@ -0,0 +1,138 @@
+use sea_orm_migration::prelude::*;
+
+use crate::drop_tables;
+
+#[derive(DeriveMigrationName)]
+pub struct Migration;
+
+#[async_trait::async_trait]
+impl MigrationTrait for Migration {
+    async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
+        manager
+            .create_table(
+                Table::create()
+                    .table(HummockSstableInfo::Table)
+                    .if_not_exists()
+                    .col(
+                        ColumnDef::new(HummockSstableInfo::SstId)
+                            .big_integer()
+                            .not_null()
+                            .primary_key(),
+                    )
+                    .col(
+                        ColumnDef::new(HummockSstableInfo::ObjectId)
+                            .big_integer()
+                            .not_null(),
+                    )
+                    .col(
+                        ColumnDef::new(HummockSstableInfo::SstableInfo)
+                            .blob(BlobSize::Long)
+                            .null(),
+                    )
+                    .to_owned(),
+            )
+            .await?;
+
+        manager
+            .create_table(
+                Table::create()
+                    .table(HummockTimeTravelVersion::Table)
+                    .if_not_exists()
+                    .col(
+                        ColumnDef::new(HummockTimeTravelVersion::VersionId)
+                            .big_integer()
+                            .not_null()
+                            .primary_key(),
+                    )
+                    .col(
+                        ColumnDef::new(HummockTimeTravelVersion::Version)
+                            .blob(BlobSize::Long)
+                            .null(),
+                    )
+                    .to_owned(),
+            )
+            .await?;
+
+        manager
+            .create_table(
+                Table::create()
+                    .table(HummockTimeTravelDelta::Table)
+                    .if_not_exists()
+                    .col(
+                        ColumnDef::new(HummockTimeTravelDelta::VersionId)
+                            .big_integer()
+                            .not_null()
+                            .primary_key(),
+                    )
+                    .col(
+                        ColumnDef::new(HummockTimeTravelDelta::VersionDelta)
+                            .blob(BlobSize::Long)
+                            .null(),
+                    )
+                    .to_owned(),
+            )
+            .await?;
+
+        manager
+            .create_table(
+                Table::create()
+                    .table(HummockEpochToVersion::Table)
+                    .if_not_exists()
+                    .col(
+                        ColumnDef::new(HummockEpochToVersion::Epoch)
+                            .big_integer()
+                            .not_null()
+                            .primary_key(),
+                    )
+                    .col(
+                        ColumnDef::new(HummockEpochToVersion::VersionId)
+                            .big_integer()
+                            .not_null(),
+                    )
+                    .to_owned(),
+            )
+            .await?;
+
+        Ok(())
+    }
+
+    async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> {
+        drop_tables!(
+            manager,
+            HummockSstableInfo,
+            HummockTimeTravelVersion,
+            HummockTimeTravelDelta,
+            HummockEpochToVersion
+        );
+        Ok(())
+    }
+}
+
+#[derive(DeriveIden)]
+enum HummockSstableInfo {
+    Table,
+    SstId,
+    ObjectId,
+    SstableInfo,
+}
+
+#[derive(DeriveIden)]
+enum HummockTimeTravelVersion {
+    Table,
+    VersionId,
+    Version,
+}
+
+#[derive(DeriveIden)]
+enum HummockTimeTravelDelta {
+    Table,
+    VersionId,
+    VersionDelta,
+}
+
+#[derive(DeriveIden)]
+enum HummockEpochToVersion {
+    Table,
+    Epoch,
+    VersionId,
+}
diff --git a/src/meta/model_v2/migration/src/m20240806_143329_add_rate_limit_to_source_catalog.rs b/src/meta/model_v2/migration/src/m20240806_143329_add_rate_limit_to_source_catalog.rs
new file mode 100644
index 0000000000000..7049e40e6d66c
--- /dev/null
+++ b/src/meta/model_v2/migration/src/m20240806_143329_add_rate_limit_to_source_catalog.rs
@@ -0,0 +1,35 @@
+use sea_orm_migration::prelude::*;
+
+#[derive(DeriveMigrationName)]
+pub struct Migration;
+
+#[async_trait::async_trait]
+impl MigrationTrait for Migration {
+    async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
+        manager
+            .alter_table(
+                Table::alter()
+                    .table(Source::Table)
+                    .add_column(ColumnDef::new(Source::RateLimit).integer())
+                    .to_owned(),
+            )
+            .await
+    }
+
+    async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> {
+        manager
+            .alter_table(
+                Table::alter()
+                    .table(Source::Table)
+                    .drop_column(Source::RateLimit)
+                    .to_owned(),
+            )
+            .await
+    }
+}
+
+#[derive(DeriveIden)]
+enum Source {
+    Table,
+    RateLimit,
+}
diff --git a/src/meta/model_v2/src/actor.rs b/src/meta/model_v2/src/actor.rs
index c75eac7dbc4cf..cbbbca543679a 100644
--- a/src/meta/model_v2/src/actor.rs
+++ b/src/meta/model_v2/src/actor.rs
@@ -56,7 +56,6 @@ pub struct Model {
     pub fragment_id: FragmentId,
     pub status: ActorStatus,
     pub splits: Option<ConnectorSplits>,
-    pub parallel_unit_id: i32,
     pub worker_id: WorkerId,
     pub upstream_actor_ids: ActorUpstreamActors,
     pub vnode_bitmap: Option<VnodeBitmap>,
diff --git a/src/meta/model_v2/src/fragment.rs b/src/meta/model_v2/src/fragment.rs
index 7f69584538593..dd332f5fc76a7 100644
--- a/src/meta/model_v2/src/fragment.rs
+++ b/src/meta/model_v2/src/fragment.rs
@@ -16,7 +16,7 @@ use risingwave_pb::meta::table_fragments::fragment::PbFragmentDistributionType;
 use sea_orm::entity::prelude::*;
 use serde::{Deserialize, Serialize};
 
-use crate::{FragmentId, FragmentVnodeMapping, I32Array, ObjectId, StreamNode};
+use crate::{FragmentId, I32Array, ObjectId, StreamNode};
 
 #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)]
 #[sea_orm(table_name = "fragment")]
@@ -27,12 +27,11 @@ pub struct Model {
     pub fragment_type_mask: i32,
     pub distribution_type: DistributionType,
     pub stream_node: StreamNode,
-    pub vnode_mapping: FragmentVnodeMapping,
     pub state_table_ids: I32Array,
     pub upstream_fragment_id: I32Array,
 }
 
-#[derive(Clone, Debug, PartialEq, Eq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)]
+#[derive(Copy, Clone, Debug, PartialEq, Eq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)]
 #[sea_orm(rs_type = "String", db_type = "String(None)")]
 pub enum DistributionType {
     #[sea_orm(string_value = "SINGLE")]
diff --git a/src/meta/model_v2/src/hummock_epoch_to_version.rs b/src/meta/model_v2/src/hummock_epoch_to_version.rs
new file mode 100644
index 0000000000000..181b1b320bc54
--- /dev/null
+++ b/src/meta/model_v2/src/hummock_epoch_to_version.rs
@@ -0,0 +1,31 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use sea_orm::entity::prelude::*;
+use sea_orm::{DeriveEntityModel, DeriveRelation, EnumIter};
+
+use crate::{Epoch, HummockVersionId};
+
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Default)]
+#[sea_orm(table_name = "hummock_epoch_to_version")]
+pub struct Model {
+    #[sea_orm(primary_key, auto_increment = false)]
+    pub epoch: Epoch,
+    pub version_id: HummockVersionId,
+}
+
+impl ActiveModelBehavior for ActiveModel {}
+
+#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
+pub enum Relation {}
diff --git a/src/meta/model_v2/src/hummock_sstable_info.rs b/src/meta/model_v2/src/hummock_sstable_info.rs
new file mode 100644
index 0000000000000..672d1f0e05a8f
--- /dev/null
+++ b/src/meta/model_v2/src/hummock_sstable_info.rs
@@ -0,0 +1,35 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use risingwave_pb::hummock::PbSstableInfo;
+use sea_orm::entity::prelude::*;
+use sea_orm::{DeriveEntityModel, DeriveRelation, EnumIter};
+
+use crate::HummockSstableObjectId;
+
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Default)]
+#[sea_orm(table_name = "hummock_sstable_info")]
+pub struct Model {
+    #[sea_orm(primary_key, auto_increment = false)]
+    pub sst_id: HummockSstableObjectId,
+    pub object_id: HummockSstableObjectId,
+    pub sstable_info: SstableInfoV2Backend,
+}
+
+impl ActiveModelBehavior for ActiveModel {}
+
+#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
+pub enum Relation {}
+
+crate::derive_from_blob!(SstableInfoV2Backend, PbSstableInfo);
diff --git a/src/meta/model_v2/src/hummock_time_travel_delta.rs b/src/meta/model_v2/src/hummock_time_travel_delta.rs
new file mode 100644
index 0000000000000..f807c6ec082fa
--- /dev/null
+++ b/src/meta/model_v2/src/hummock_time_travel_delta.rs
@@ -0,0 +1,34 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use risingwave_pb::hummock::PbHummockVersionDelta;
+use sea_orm::entity::prelude::*;
+use sea_orm::{DeriveEntityModel, DeriveRelation, EnumIter};
+
+use crate::HummockVersionId;
+
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Default)]
+#[sea_orm(table_name = "hummock_time_travel_delta")]
+pub struct Model {
+    #[sea_orm(primary_key, auto_increment = false)]
+    pub version_id: HummockVersionId,
+    pub version_delta: HummockVersionDelta,
+}
+
+impl ActiveModelBehavior for ActiveModel {}
+
+#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
+pub enum Relation {}
+
+crate::derive_from_blob!(HummockVersionDelta, PbHummockVersionDelta);
diff --git a/src/meta/model_v2/src/hummock_time_travel_version.rs b/src/meta/model_v2/src/hummock_time_travel_version.rs
new file mode 100644
index 0000000000000..91eb42fb52096
--- /dev/null
+++ b/src/meta/model_v2/src/hummock_time_travel_version.rs
@@ -0,0 +1,34 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use risingwave_pb::hummock::PbHummockVersion;
+use sea_orm::entity::prelude::*;
+use sea_orm::{DeriveEntityModel, DeriveRelation, EnumIter};
+
+use crate::HummockVersionId;
+
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Default)]
+#[sea_orm(table_name = "hummock_time_travel_version")]
+pub struct Model {
+    #[sea_orm(primary_key, auto_increment = false)]
+    pub version_id: HummockVersionId,
+    pub version: HummockVersion,
+}
+
+impl ActiveModelBehavior for ActiveModel {}
+
+#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
+pub enum Relation {}
+
+crate::derive_from_blob!(HummockVersion, PbHummockVersion);
diff --git a/src/meta/model_v2/src/lib.rs b/src/meta/model_v2/src/lib.rs
index 751ae99b64a17..23f97e7778519 100644
--- a/src/meta/model_v2/src/lib.rs
+++ b/src/meta/model_v2/src/lib.rs
@@ -35,9 +35,13 @@ pub mod connection;
 pub mod database;
 pub mod fragment;
 pub mod function;
+pub mod hummock_epoch_to_version;
 pub mod hummock_pinned_snapshot;
 pub mod hummock_pinned_version;
 pub mod hummock_sequence;
+pub mod hummock_sstable_info;
+pub mod hummock_time_travel_delta;
+pub mod hummock_time_travel_version;
 pub mod hummock_version_delta;
 pub mod hummock_version_stats;
 pub mod index;
@@ -393,9 +397,10 @@ derive_from_blob!(VnodeBitmap, risingwave_pb::common::Buffer);
 derive_from_blob!(ActorMapping, risingwave_pb::stream_plan::PbActorMapping);
 derive_from_blob!(ExprContext, risingwave_pb::plan_common::PbExprContext);
 
-derive_from_blob!(
-    FragmentVnodeMapping,
-    risingwave_pb::common::ParallelUnitMapping
+derive_array_from_blob!(
+    HummockVersionDeltaArray,
+    risingwave_pb::hummock::PbHummockVersionDelta,
+    PbHummockVersionDeltaArray
 );
 
 #[derive(Clone, Debug, PartialEq, FromJsonQueryResult, Serialize, Deserialize)]
diff --git a/src/meta/model_v2/src/sink.rs b/src/meta/model_v2/src/sink.rs
index ad72aa3df981c..21aea7b696f4d 100644
--- a/src/meta/model_v2/src/sink.rs
+++ b/src/meta/model_v2/src/sink.rs
@@ -74,6 +74,7 @@ pub struct Model {
     pub target_table: Option<TableId>,
     // `secret_ref` stores the mapping info mapping from property name to secret id and type.
     pub secret_ref: Option<SecretRef>,
+    pub original_target_columns: ColumnCatalogArray,
 }
 
 #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
@@ -130,6 +131,7 @@ impl From<PbSink> for ActiveModel {
             sink_format_desc: Set(pb_sink.format_desc.as_ref().map(|x| x.into())),
             target_table: Set(pb_sink.target_table.map(|x| x as _)),
             secret_ref: Set(Some(SecretRef::from(pb_sink.secret_refs))),
+            original_target_columns: Set(pb_sink.original_target_columns.into()),
         }
     }
 }
diff --git a/src/meta/model_v2/src/source.rs b/src/meta/model_v2/src/source.rs
index 9dd4bb5b1a11a..39c7dc556cf3e 100644
--- a/src/meta/model_v2/src/source.rs
+++ b/src/meta/model_v2/src/source.rs
@@ -41,6 +41,7 @@ pub struct Model {
     pub version: i64,
     // `secret_ref` stores the mapping info mapping from property name to secret id and type.
     pub secret_ref: Option<SecretRef>,
+    pub rate_limit: Option<i32>,
 }
 
 #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
@@ -104,6 +105,7 @@ impl From<PbSource> for ActiveModel {
             connection_id: Set(source.connection_id.map(|id| id as _)),
             version: Set(source.version as _),
             secret_ref: Set(Some(SecretRef::from(source.secret_refs))),
+            rate_limit: Set(source.rate_limit.map(|id| id as _)),
         }
     }
 }
diff --git a/src/meta/model_v2/src/worker_property.rs b/src/meta/model_v2/src/worker_property.rs
index 3ab8d411c8b5c..64834ae0b13cb 100644
--- a/src/meta/model_v2/src/worker_property.rs
+++ b/src/meta/model_v2/src/worker_property.rs
@@ -15,14 +15,14 @@
 use sea_orm::entity::prelude::*;
 use serde::{Deserialize, Serialize};
 
-use crate::{I32Array, WorkerId};
+use crate::WorkerId;
 
 #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)]
 #[sea_orm(table_name = "worker_property")]
 pub struct Model {
     #[sea_orm(primary_key, auto_increment = false)]
     pub worker_id: WorkerId,
-    pub parallel_unit_ids: I32Array,
+    pub parallelism: i32,
     pub is_streaming: bool,
     pub is_serving: bool,
     pub is_unschedulable: bool,
diff --git a/src/meta/node/Cargo.toml b/src/meta/node/Cargo.toml
index d0ab59edee998..9300a08727b40 100644
--- a/src/meta/node/Cargo.toml
+++ b/src/meta/node/Cargo.toml
@@ -16,9 +16,11 @@ normal = ["workspace-hack"]
 [dependencies]
 anyhow = "1"
 clap = { workspace = true }
+educe = "0.6"
 either = "1"
 etcd-client = { workspace = true }
 futures = { version = "0.3", default-features = false, features = ["alloc"] }
+hex = "0.4"
 itertools = { workspace = true }
 otlp-embedded = { workspace = true }
 prometheus-http-query = "0.8"
diff --git a/src/meta/node/src/lib.rs b/src/meta/node/src/lib.rs
index a7600ba930a15..65e730166aed8 100644
--- a/src/meta/node/src/lib.rs
+++ b/src/meta/node/src/lib.rs
@@ -21,11 +21,13 @@ mod server;
 use std::time::Duration;
 
 use clap::Parser;
+use educe::Educe;
 pub use error::{MetaError, MetaResult};
 use redact::Secret;
 use risingwave_common::config::OverrideConfig;
 use risingwave_common::util::meta_addr::MetaAddressStrategy;
 use risingwave_common::util::resource_util;
+use risingwave_common::util::tokio_util::sync::CancellationToken;
 use risingwave_common::{GIT_SHA, RW_VERSION};
 use risingwave_common_heap_profiling::HeapProfiler;
 use risingwave_meta::*;
@@ -36,7 +38,8 @@ pub use server::started::get as is_server_started;
 
 use crate::manager::MetaOpts;
 
-#[derive(Debug, Clone, Parser, OverrideConfig)]
+#[derive(Educe, Clone, Parser, OverrideConfig)]
+#[educe(Debug)]
 #[command(version, about = "The central metadata management service")]
 pub struct MetaNodeOpts {
     // TODO: use `SocketAddr`
@@ -183,6 +186,20 @@ pub struct MetaNodeOpts {
     #[deprecated = "connector node has been deprecated."]
     #[clap(long, hide = true, env = "RW_CONNECTOR_RPC_ENDPOINT")]
     pub connector_rpc_endpoint: Option<String>,
+
+    /// 128-bit AES key for secret store in HEX format.
+    #[educe(Debug(ignore))]
+    #[clap(long, hide = true, env = "RW_SECRET_STORE_PRIVATE_KEY_HEX")]
+    pub secret_store_private_key_hex: Option<String>,
+
+    /// The path of the temp secret file directory.
+    #[clap(
+        long,
+        hide = true,
+        env = "RW_TEMP_SECRET_FILE_DIR",
+        default_value = "./secrets"
+    )]
+    pub temp_secret_file_dir: String,
 }
 
 impl risingwave_common::opts::Opts for MetaNodeOpts {
@@ -204,7 +221,10 @@ use risingwave_common::config::{load_config, MetaBackend, RwConfig};
 use tracing::info;
 
 /// Start meta node
-pub fn start(opts: MetaNodeOpts) -> Pin<Box<dyn Future<Output = ()> + Send>> {
+pub fn start(
+    opts: MetaNodeOpts,
+    shutdown: CancellationToken,
+) -> Pin<Box<dyn Future<Output = ()> + Send>> {
     // WARNING: don't change the function signature. Making it `async fn` will cause
     // slow compile in release mode.
     Box::pin(async move {
@@ -279,6 +299,9 @@ pub fn start(opts: MetaNodeOpts) -> Pin<Box<dyn Future<Output = ()> + Send>> {
         // Run a background heap profiler
         heap_profiler.start();
 
+        let secret_store_private_key = opts
+            .secret_store_private_key_hex
+            .map(|key| hex::decode(key).unwrap());
         let max_heartbeat_interval =
             Duration::from_secs(config.meta.max_heartbeat_interval_secs as u64);
         let max_idle_ms = config.meta.dangerous_max_idle_secs.unwrap_or(0) * 1000;
@@ -324,7 +347,7 @@ pub fn start(opts: MetaNodeOpts) -> Pin<Box<dyn Future<Output = ()> + Send>> {
             max_timeout_ms / 1000
         } + MIN_TIMEOUT_INTERVAL_SEC;
 
-        let (mut join_handle, leader_lost_handle, shutdown_send) = rpc_serve(
+        rpc_serve(
             add_info,
             backend,
             max_heartbeat_interval,
@@ -351,6 +374,9 @@ pub fn start(opts: MetaNodeOpts) -> Pin<Box<dyn Future<Output = ()> + Send>> {
                     .meta
                     .hummock_version_checkpoint_interval_sec,
                 enable_hummock_data_archive: config.meta.enable_hummock_data_archive,
+                hummock_time_travel_snapshot_interval: config
+                    .meta
+                    .hummock_time_travel_snapshot_interval,
                 min_delta_log_num_for_hummock_version_checkpoint: config
                     .meta
                     .min_delta_log_num_for_hummock_version_checkpoint,
@@ -421,49 +447,18 @@ pub fn start(opts: MetaNodeOpts) -> Pin<Box<dyn Future<Output = ()> + Send>> {
                     .developer
                     .max_trivial_move_task_count_per_loop,
                 max_get_task_probe_times: config.meta.developer.max_get_task_probe_times,
-                secret_store_private_key: config.meta.secret_store_private_key,
+                secret_store_private_key,
+                temp_secret_file_dir: opts.temp_secret_file_dir,
                 table_info_statistic_history_times: config
                     .storage
                     .table_info_statistic_history_times,
             },
             config.system.into_init_system_params(),
             Default::default(),
+            shutdown,
         )
         .await
         .unwrap();
-
-        tracing::info!("Meta server listening at {}", listen_addr);
-
-        match leader_lost_handle {
-            None => {
-                tokio::select! {
-                    _ = tokio::signal::ctrl_c() => {
-                        tracing::info!("receive ctrl+c");
-                        shutdown_send.send(()).unwrap();
-                        join_handle.await.unwrap()
-                    }
-                    res = &mut join_handle => res.unwrap(),
-                };
-            }
-            Some(mut handle) => {
-                tokio::select! {
-                    _ = &mut handle => {
-                        tracing::info!("receive leader lost signal");
-                        // When we lose leadership, we will exit as soon as possible.
-                    }
-                    _ = tokio::signal::ctrl_c() => {
-                        tracing::info!("receive ctrl+c");
-                        shutdown_send.send(()).unwrap();
-                        join_handle.await.unwrap();
-                        handle.abort();
-                    }
-                    res = &mut join_handle => {
-                        res.unwrap();
-                        handle.abort();
-                    },
-                };
-            }
-        };
     })
 }
 
diff --git a/src/meta/node/src/server.rs b/src/meta/node/src/server.rs
index 57ccad6c9b5e6..5c72b39bcd156 100644
--- a/src/meta/node/src/server.rs
+++ b/src/meta/node/src/server.rs
@@ -16,22 +16,24 @@ use std::sync::Arc;
 use std::time::Duration;
 
 use anyhow::Context;
-use either::Either;
 use etcd_client::ConnectOptions;
-use futures::future::join_all;
 use otlp_embedded::TraceServiceServer;
 use regex::Regex;
-use risingwave_common::monitor::connection::{RouterExt, TcpConfig};
+use risingwave_common::monitor::{RouterExt, TcpConfig};
+use risingwave_common::secret::LocalSecretManager;
 use risingwave_common::session_config::SessionConfig;
 use risingwave_common::system_param::reader::SystemParamsRead;
 use risingwave_common::telemetry::manager::TelemetryManager;
 use risingwave_common::telemetry::{report_scarf_enabled, report_to_scarf, telemetry_env_enabled};
-use risingwave_common_service::metrics_manager::MetricsManager;
-use risingwave_common_service::tracing::TracingExtractLayer;
+use risingwave_common::util::tokio_util::sync::CancellationToken;
+use risingwave_common_service::{MetricsManager, TracingExtractLayer};
 use risingwave_meta::barrier::StreamRpcManager;
 use risingwave_meta::controller::catalog::CatalogController;
 use risingwave_meta::controller::cluster::ClusterController;
-use risingwave_meta::manager::{MetaStoreImpl, MetadataManager, SystemParamsManagerImpl};
+use risingwave_meta::manager::{
+    MetaStoreImpl, MetadataManager, SystemParamsManagerImpl, META_NODE_ID,
+};
+use risingwave_meta::rpc::election::dummy::DummyElectionClient;
 use risingwave_meta::rpc::intercept::MetricsMiddlewareLayer;
 use risingwave_meta::rpc::ElectionClientRef;
 use risingwave_meta::stream::ScaleController;
@@ -77,10 +79,7 @@ use risingwave_pb::user::user_service_server::UserServiceServer;
 use risingwave_rpc_client::ComputeClientPool;
 use sea_orm::{ConnectionTrait, DbBackend};
 use thiserror_ext::AsReport;
-use tokio::sync::oneshot::{channel as OneChannel, Receiver as OneReceiver};
 use tokio::sync::watch;
-use tokio::sync::watch::{Receiver as WatchReceiver, Sender as WatchSender};
-use tokio::task::JoinHandle;
 
 use crate::backup_restore::BackupManager;
 use crate::barrier::{BarrierScheduler, GlobalBarrierManager};
@@ -125,6 +124,9 @@ pub mod started {
     }
 }
 
+/// A wrapper around [`rpc_serve_with_store`] that dispatches different store implementations.
+///
+/// For the timing of returning, see [`rpc_serve_with_store`].
 pub async fn rpc_serve(
     address_info: AddressInfo,
     meta_store_backend: MetaStoreBackend,
@@ -133,7 +135,8 @@ pub async fn rpc_serve(
     opts: MetaOpts,
     init_system_params: SystemParams,
     init_session_config: SessionConfig,
-) -> MetaResult<(JoinHandle<()>, Option<JoinHandle<()>>, WatchSender<()>)> {
+    shutdown: CancellationToken,
+) -> MetaResult<()> {
     match meta_store_backend {
         MetaStoreBackend::Etcd {
             endpoints,
@@ -169,27 +172,34 @@ pub async fn rpc_serve(
 
             rpc_serve_with_store(
                 MetaStoreImpl::Kv(meta_store),
-                Some(election_client),
+                election_client,
                 address_info,
                 max_cluster_heartbeat_interval,
                 lease_interval_secs,
                 opts,
                 init_system_params,
                 init_session_config,
+                shutdown,
             )
+            .await
         }
         MetaStoreBackend::Mem => {
             let meta_store = MemStore::new().into_ref();
+            let dummy_election_client = Arc::new(DummyElectionClient::new(
+                address_info.advertise_addr.clone(),
+            ));
             rpc_serve_with_store(
                 MetaStoreImpl::Kv(meta_store),
-                None,
+                dummy_election_client,
                 address_info,
                 max_cluster_heartbeat_interval,
                 lease_interval_secs,
                 opts,
                 init_system_params,
                 init_session_config,
+                shutdown,
             )
+            .await
         }
         MetaStoreBackend::Sql { endpoint } => {
             let max_connection = if DbBackend::Sqlite.is_prefix_of(&endpoint) {
@@ -226,130 +236,120 @@ pub async fn rpc_serve(
 
             rpc_serve_with_store(
                 MetaStoreImpl::Sql(meta_store_sql),
-                Some(election_client),
+                election_client,
                 address_info,
                 max_cluster_heartbeat_interval,
                 lease_interval_secs,
                 opts,
                 init_system_params,
                 init_session_config,
+                shutdown,
             )
+            .await
         }
     }
 }
 
-#[expect(clippy::type_complexity)]
-pub fn rpc_serve_with_store(
+/// Bootstraps the follower or leader service based on the election status.
+///
+/// Returns when the `shutdown` token is triggered, or when leader status is lost, or if the leader
+/// service fails to start.
+pub async fn rpc_serve_with_store(
     meta_store_impl: MetaStoreImpl,
-    election_client: Option<ElectionClientRef>,
+    election_client: ElectionClientRef,
     address_info: AddressInfo,
     max_cluster_heartbeat_interval: Duration,
     lease_interval_secs: u64,
     opts: MetaOpts,
     init_system_params: SystemParams,
     init_session_config: SessionConfig,
-) -> MetaResult<(JoinHandle<()>, Option<JoinHandle<()>>, WatchSender<()>)> {
-    let (svc_shutdown_tx, svc_shutdown_rx) = watch::channel(());
+    shutdown: CancellationToken,
+) -> MetaResult<()> {
+    // TODO(shutdown): directly use cancellation token
+    let (election_shutdown_tx, election_shutdown_rx) = watch::channel(());
 
-    let leader_lost_handle = if let Some(election_client) = election_client.clone() {
-        let stop_rx = svc_shutdown_tx.subscribe();
+    let election_handle = tokio::spawn({
+        let shutdown = shutdown.clone();
+        let election_client = election_client.clone();
 
-        let handle = tokio::spawn(async move {
+        async move {
             while let Err(e) = election_client
-                .run_once(lease_interval_secs as i64, stop_rx.clone())
+                .run_once(lease_interval_secs as i64, election_shutdown_rx.clone())
                 .await
             {
                 tracing::error!(error = %e.as_report(), "election error happened");
             }
-        });
+            // Leader lost, shutdown the service.
+            shutdown.cancel();
+        }
+    });
 
-        Some(handle)
-    } else {
-        None
-    };
+    // Spawn and run the follower service if not the leader.
+    // Watch the leader status and switch to the leader service when elected.
+    // TODO: the branch seems to be always hit since the default value of `is_leader` is false until
+    // the election is done (unless using `DummyElectionClient`).
+    if !election_client.is_leader() {
+        // The follower service can be shutdown separately if we're going to be the leader.
+        let follower_shutdown = shutdown.child_token();
+
+        let follower_handle = tokio::spawn(start_service_as_election_follower(
+            follower_shutdown.clone(),
+            address_info.clone(),
+            election_client.clone(),
+        ));
 
-    let join_handle = tokio::spawn(async move {
-        if let Some(election_client) = election_client.clone() {
-            let mut is_leader_watcher = election_client.subscribe();
-            let mut svc_shutdown_rx_clone = svc_shutdown_rx.clone();
-            let (follower_shutdown_tx, follower_shutdown_rx) = OneChannel::<()>();
+        // Watch and wait until we become the leader.
+        let mut is_leader_watcher = election_client.subscribe();
 
+        while !*is_leader_watcher.borrow_and_update() {
             tokio::select! {
-                _ = svc_shutdown_rx_clone.changed() => return,
+                // External shutdown signal. Directly return without switching to leader.
+                _ = shutdown.cancelled() => return Ok(()),
+
                 res = is_leader_watcher.changed() => {
                     if res.is_err() {
                         tracing::error!("leader watcher recv failed");
                     }
                 }
             }
-            let svc_shutdown_rx_clone = svc_shutdown_rx.clone();
-
-            // If not the leader, spawn a follower.
-            let follower_handle: Option<JoinHandle<()>> = if !*is_leader_watcher.borrow() {
-                let address_info_clone = address_info.clone();
-
-                let election_client_ = election_client.clone();
-                Some(tokio::spawn(async move {
-                    start_service_as_election_follower(
-                        svc_shutdown_rx_clone,
-                        follower_shutdown_rx,
-                        address_info_clone,
-                        Some(election_client_),
-                    )
-                    .await;
-                }))
-            } else {
-                None
-            };
+        }
 
-            let mut svc_shutdown_rx_clone = svc_shutdown_rx.clone();
-            while !*is_leader_watcher.borrow_and_update() {
-                tokio::select! {
-                    _ = svc_shutdown_rx_clone.changed() => {
-                        return;
-                    }
-                    res = is_leader_watcher.changed() => {
-                        if res.is_err() {
-                            tracing::error!("leader watcher recv failed");
-                        }
-                    }
-                }
-            }
+        tracing::info!("elected as leader, shutting down follower services");
+        follower_shutdown.cancel();
+        let _ = follower_handle.await;
+    }
 
-            if let Some(handle) = follower_handle {
-                let _res = follower_shutdown_tx.send(());
-                let _ = handle.await;
-            }
-        };
+    // Run the leader service.
+    let result = start_service_as_election_leader(
+        meta_store_impl,
+        address_info,
+        max_cluster_heartbeat_interval,
+        opts,
+        init_system_params,
+        init_session_config,
+        election_client,
+        shutdown,
+    )
+    .await;
 
-        start_service_as_election_leader(
-            meta_store_impl,
-            address_info,
-            max_cluster_heartbeat_interval,
-            opts,
-            init_system_params,
-            init_session_config,
-            election_client,
-            svc_shutdown_rx,
-        )
-        .await
-        .expect("Unable to start leader services");
-    });
+    // Leader service has stopped, shutdown the election service to gracefully resign.
+    election_shutdown_tx.send(()).ok();
+    let _ = election_handle.await;
 
-    Ok((join_handle, leader_lost_handle, svc_shutdown_tx))
+    result
 }
 
-/// Starts all services needed for the meta follower node
+/// Starts all services needed for the meta follower node.
+///
+/// Returns when the `shutdown` token is triggered.
 pub async fn start_service_as_election_follower(
-    mut svc_shutdown_rx: WatchReceiver<()>,
-    follower_shutdown_rx: OneReceiver<()>,
+    shutdown: CancellationToken,
     address_info: AddressInfo,
-    election_client: Option<ElectionClientRef>,
+    election_client: ElectionClientRef,
 ) {
-    let meta_member_srv = MetaMemberServiceImpl::new(match election_client {
-        None => Either::Right(address_info.clone()),
-        Some(election_client) => Either::Left(election_client),
-    });
+    tracing::info!("starting follower services");
+
+    let meta_member_srv = MetaMemberServiceImpl::new(election_client);
 
     let health_srv = HealthServiceImpl::new();
 
@@ -367,35 +367,21 @@ pub async fn start_service_as_election_follower(
                 tcp_nodelay: true,
                 keepalive_duration: None,
             },
-            async move {
-                tokio::select! {
-                    // shutdown service if all services should be shut down
-                    res = svc_shutdown_rx.changed() => {
-                        match res {
-                            Ok(_) => tracing::info!("Shutting down services"),
-                            Err(_) => tracing::error!("Service shutdown sender dropped")
-                        }
-                    },
-                    // shutdown service if follower becomes leader
-                    res = follower_shutdown_rx => {
-                        match res {
-                            Ok(_) => tracing::info!("Shutting down follower services"),
-                            Err(_) => tracing::error!("Follower service shutdown sender dropped")
-                        }
-                    },
-                }
-            },
+            shutdown.clone().cancelled_owned(),
         );
+    let server_handle = tokio::spawn(server);
     started::set();
-    server.await;
+
+    // Wait for the shutdown signal.
+    shutdown.cancelled().await;
+    // Wait for the server to shutdown. This is necessary because we may be transitioning from follower
+    // to leader, and conflicts on the services must be avoided.
+    let _ = server_handle.await;
 }
 
-/// Starts all services needed for the meta leader node
-/// Only call this function once, since initializing the services multiple times will result in an
-/// inconsistent state
+/// Starts all services needed for the meta leader node.
 ///
-/// ## Returns
-/// Returns an error if the service initialization failed
+/// Returns when the `shutdown` token is triggered, or if the service initialization fails.
 pub async fn start_service_as_election_leader(
     meta_store_impl: MetaStoreImpl,
     address_info: AddressInfo,
@@ -403,10 +389,11 @@ pub async fn start_service_as_election_leader(
     opts: MetaOpts,
     init_system_params: SystemParams,
     init_session_config: SessionConfig,
-    election_client: Option<ElectionClientRef>,
-    mut svc_shutdown_rx: WatchReceiver<()>,
+    election_client: ElectionClientRef,
+    shutdown: CancellationToken,
 ) -> MetaResult<()> {
-    tracing::info!("Defining leader services");
+    tracing::info!("starting leader services");
+
     let env = MetaSrvEnv::new(
         opts.clone(),
         init_system_params,
@@ -480,10 +467,7 @@ pub async fn start_service_as_election_leader(
     .unwrap();
     let object_store_media_type = hummock_manager.object_store_media_type();
 
-    let meta_member_srv = MetaMemberServiceImpl::new(match election_client.clone() {
-        None => Either::Right(address_info.clone()),
-        Some(election_client) => Either::Left(election_client),
-    });
+    let meta_member_srv = MetaMemberServiceImpl::new(election_client.clone());
 
     let prometheus_client = opts.prometheus_endpoint.as_ref().map(|x| {
         use std::str::FromStr;
@@ -505,13 +489,13 @@ pub async fn start_service_as_election_leader(
     let trace_srv = otlp_embedded::TraceServiceImpl::new(trace_state.clone());
 
     #[cfg(not(madsim))]
-    let dashboard_task = if let Some(ref dashboard_addr) = address_info.dashboard_addr {
+    let _dashboard_task = if let Some(ref dashboard_addr) = address_info.dashboard_addr {
         let dashboard_service = crate::dashboard::DashboardService {
             dashboard_addr: *dashboard_addr,
             prometheus_client,
             prometheus_selector,
             metadata_manager: metadata_manager.clone(),
-            compute_clients: ComputeClientPool::default(),
+            compute_clients: ComputeClientPool::new(1), // typically no need for plural clients
             diagnose_command,
             trace_state,
         };
@@ -527,6 +511,31 @@ pub async fn start_service_as_election_leader(
         system_params_reader.checkpoint_frequency() as usize,
     );
 
+    // Initialize services.
+    let backup_manager = BackupManager::new(
+        env.clone(),
+        hummock_manager.clone(),
+        meta_metrics.clone(),
+        system_params_reader.backup_storage_url(),
+        system_params_reader.backup_storage_directory(),
+    )
+    .await?;
+
+    LocalSecretManager::init(
+        opts.temp_secret_file_dir,
+        env.cluster_id().to_string(),
+        META_NODE_ID,
+    );
+
+    let notification_srv = NotificationServiceImpl::new(
+        env.clone(),
+        metadata_manager.clone(),
+        hummock_manager.clone(),
+        backup_manager.clone(),
+        serving_vnode_mapping.clone(),
+    )
+    .await?;
+
     let source_manager = Arc::new(
         SourceManager::new(
             barrier_scheduler.clone(),
@@ -538,6 +547,7 @@ pub async fn start_service_as_election_leader(
     );
 
     let (sink_manager, shutdown_handle) = SinkCoordinatorManager::start_worker();
+    // TODO(shutdown): remove this as there's no need to gracefully shutdown some of these sub-tasks.
     let mut sub_tasks = vec![shutdown_handle];
 
     let stream_rpc_manager = StreamRpcManager::new(env.clone());
@@ -559,7 +569,8 @@ pub async fn start_service_as_election_leader(
         meta_metrics.clone(),
         stream_rpc_manager.clone(),
         scale_controller.clone(),
-    );
+    )
+    .await;
 
     {
         let source_manager = source_manager.clone();
@@ -585,15 +596,6 @@ pub async fn start_service_as_election_leader(
         .await
         .unwrap();
 
-    // Initialize services.
-    let backup_manager = BackupManager::new(
-        env.clone(),
-        hummock_manager.clone(),
-        meta_metrics.clone(),
-        system_params_reader.backup_storage_url(),
-        system_params_reader.backup_storage_directory(),
-    )
-    .await?;
     let vacuum_manager = Arc::new(hummock::VacuumManager::new(
         env.clone(),
         hummock_manager.clone(),
@@ -630,7 +632,8 @@ pub async fn start_service_as_election_leader(
         scale_controller.clone(),
     );
 
-    let cluster_srv = ClusterServiceImpl::new(metadata_manager.clone());
+    let cluster_srv =
+        ClusterServiceImpl::new(metadata_manager.clone(), barrier_manager.context().clone());
     let stream_srv = StreamServiceImpl::new(
         env.clone(),
         barrier_scheduler.clone(),
@@ -643,13 +646,7 @@ pub async fn start_service_as_election_leader(
         vacuum_manager.clone(),
         metadata_manager.clone(),
     );
-    let notification_srv = NotificationServiceImpl::new(
-        env.clone(),
-        metadata_manager.clone(),
-        hummock_manager.clone(),
-        backup_manager.clone(),
-        serving_vnode_mapping.clone(),
-    );
+
     let health_srv = HealthServiceImpl::new();
     let backup_srv = BackupServiceImpl::new(backup_manager);
     let telemetry_srv = TelemetryInfoServiceImpl::new(env.meta_store());
@@ -722,17 +719,17 @@ pub async fn start_service_as_election_leader(
             sub_tasks.push(stream_manager.start_auto_parallelism_monitor());
         }
     }
-    let (idle_send, idle_recv) = tokio::sync::oneshot::channel();
-    sub_tasks.push(IdleManager::start_idle_checker(
+
+    let _idle_checker_handle = IdleManager::start_idle_checker(
         env.idle_manager_ref(),
         Duration::from_secs(30),
-        idle_send,
-    ));
+        shutdown.clone(),
+    );
 
     let (abort_sender, abort_recv) = tokio::sync::oneshot::channel();
     let notification_mgr = env.notification_manager_ref();
     let stream_abort_handler = tokio::spawn(async move {
-        abort_recv.await.unwrap();
+        let _ = abort_recv.await;
         notification_mgr.abort_all().await;
         compactor_manager.abort_all_compactors();
     });
@@ -763,33 +760,6 @@ pub async fn start_service_as_election_leader(
         sub_tasks.push(pair);
     }
 
-    let shutdown_all = async move {
-        let mut handles = Vec::with_capacity(sub_tasks.len());
-
-        for (join_handle, shutdown_sender) in sub_tasks {
-            if let Err(_err) = shutdown_sender.send(()) {
-                continue;
-            }
-
-            handles.push(join_handle);
-        }
-
-        // The barrier manager can't be shutdown gracefully if it's under recovering, try to
-        // abort it using timeout.
-        match tokio::time::timeout(Duration::from_secs(1), join_all(handles)).await {
-            Ok(results) => {
-                for result in results {
-                    if result.is_err() {
-                        tracing::warn!("Failed to join shutdown");
-                    }
-                }
-            }
-            Err(_e) => {
-                tracing::warn!("Join shutdown timeout");
-            }
-        }
-    };
-
     tracing::info!("Assigned cluster id {:?}", *env.cluster_id());
     tracing::info!("Starting meta services");
 
@@ -833,28 +803,15 @@ pub async fn start_service_as_election_leader(
                 tcp_nodelay: true,
                 keepalive_duration: None,
             },
-            async move {
-                tokio::select! {
-                    res = svc_shutdown_rx.changed() => {
-                        match res {
-                            Ok(_) => tracing::info!("Shutting down services"),
-                            Err(_) => tracing::error!("Service shutdown receiver dropped")
-                        }
-                        shutdown_all.await;
-                    },
-                    _ = idle_recv => {
-                        shutdown_all.await;
-                    },
-                }
-            },
+            shutdown.clone().cancelled_owned(),
         );
     started::set();
-    server.await;
+    let _server_handle = tokio::spawn(server);
 
-    #[cfg(not(madsim))]
-    if let Some(dashboard_task) = dashboard_task {
-        dashboard_task.abort();
-    }
+    // Wait for the shutdown signal.
+    shutdown.cancelled().await;
+    // TODO(shutdown): may warn user if there's any other node still running in the cluster.
+    // TODO(shutdown): do we have any other shutdown tasks?
     Ok(())
 }
 
diff --git a/src/meta/service/Cargo.toml b/src/meta/service/Cargo.toml
index 6e00e6eb4318e..1e3330a2b53a0 100644
--- a/src/meta/service/Cargo.toml
+++ b/src/meta/service/Cargo.toml
@@ -19,6 +19,7 @@ async-trait = "0.1"
 either = "1"
 futures = { version = "0.3", default-features = false, features = ["alloc"] }
 itertools = { workspace = true }
+prost = { workspace = true }
 rand = { workspace = true }
 regex = "1"
 risingwave_common = { workspace = true }
diff --git a/src/meta/service/src/cloud_service.rs b/src/meta/service/src/cloud_service.rs
index 5777f1c405009..739d7dc2d1973 100644
--- a/src/meta/service/src/cloud_service.rs
+++ b/src/meta/service/src/cloud_service.rs
@@ -17,12 +17,12 @@ use std::sync::LazyLock;
 
 use async_trait::async_trait;
 use regex::Regex;
-use risingwave_connector::dispatch_source_prop;
 use risingwave_connector::error::ConnectorResult;
 use risingwave_connector::source::kafka::private_link::insert_privatelink_broker_rewrite_map;
 use risingwave_connector::source::{
     ConnectorProperties, SourceEnumeratorContext, SourceProperties, SplitEnumerator,
 };
+use risingwave_connector::{dispatch_source_prop, WithOptionsSecResolved};
 use risingwave_meta::manager::{ConnectionId, MetadataManager};
 use risingwave_pb::catalog::connection::Info::PrivateLinkService;
 use risingwave_pb::cloud_service::cloud_service_server::CloudService;
@@ -146,6 +146,10 @@ impl CloudService for CloudServiceImpl {
                 ));
             }
         }
+
+        // XXX: We can't use secret in cloud validate source.
+        let source_cfg = WithOptionsSecResolved::without_secrets(source_cfg);
+
         // try fetch kafka metadata, return error message on failure
         let props = ConnectorProperties::extract(source_cfg, false);
         if let Err(e) = props {
diff --git a/src/meta/service/src/cluster_service.rs b/src/meta/service/src/cluster_service.rs
index ef7295a6d5136..39cd40ed37400 100644
--- a/src/meta/service/src/cluster_service.rs
+++ b/src/meta/service/src/cluster_service.rs
@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use risingwave_meta::barrier::BarrierManagerRef;
 use risingwave_meta::manager::MetadataManager;
 use risingwave_meta_model_v2::WorkerId;
 use risingwave_pb::common::worker_node::State;
@@ -19,7 +20,8 @@ use risingwave_pb::common::HostAddress;
 use risingwave_pb::meta::cluster_service_server::ClusterService;
 use risingwave_pb::meta::{
     ActivateWorkerNodeRequest, ActivateWorkerNodeResponse, AddWorkerNodeRequest,
-    AddWorkerNodeResponse, DeleteWorkerNodeRequest, DeleteWorkerNodeResponse, ListAllNodesRequest,
+    AddWorkerNodeResponse, DeleteWorkerNodeRequest, DeleteWorkerNodeResponse,
+    GetClusterRecoveryStatusRequest, GetClusterRecoveryStatusResponse, ListAllNodesRequest,
     ListAllNodesResponse, UpdateWorkerNodeSchedulabilityRequest,
     UpdateWorkerNodeSchedulabilityResponse,
 };
@@ -31,11 +33,15 @@ use crate::MetaError;
 #[derive(Clone)]
 pub struct ClusterServiceImpl {
     metadata_manager: MetadataManager,
+    barrier_manager: BarrierManagerRef,
 }
 
 impl ClusterServiceImpl {
-    pub fn new(metadata_manager: MetadataManager) -> Self {
-        ClusterServiceImpl { metadata_manager }
+    pub fn new(metadata_manager: MetadataManager, barrier_manager: BarrierManagerRef) -> Self {
+        ClusterServiceImpl {
+            metadata_manager,
+            barrier_manager,
+        }
     }
 }
 
@@ -56,10 +62,12 @@ impl ClusterService for ClusterServiceImpl {
             .metadata_manager
             .add_worker_node(worker_type, host, property, resource)
             .await;
+        let cluster_id = self.metadata_manager.cluster_id().to_string();
         match result {
             Ok(worker_id) => Ok(Response::new(AddWorkerNodeResponse {
                 status: None,
                 node_id: Some(worker_id),
+                cluster_id,
             })),
             Err(e) => {
                 if e.is_invalid_worker() {
@@ -69,6 +77,7 @@ impl ClusterService for ClusterServiceImpl {
                             message: e.to_report_string(),
                         }),
                         node_id: None,
+                        cluster_id,
                     }));
                 }
                 Err(e.into())
@@ -176,4 +185,13 @@ impl ClusterService for ClusterServiceImpl {
             nodes: node_list,
         }))
     }
+
+    async fn get_cluster_recovery_status(
+        &self,
+        _request: Request<GetClusterRecoveryStatusRequest>,
+    ) -> Result<Response<GetClusterRecoveryStatusResponse>, Status> {
+        Ok(Response::new(GetClusterRecoveryStatusResponse {
+            status: self.barrier_manager.get_recovery_status() as _,
+        }))
+    }
 }
diff --git a/src/meta/service/src/ddl_service.rs b/src/meta/service/src/ddl_service.rs
index 2e4ba23e02d8f..2594ccc123ff3 100644
--- a/src/meta/service/src/ddl_service.rs
+++ b/src/meta/service/src/ddl_service.rs
@@ -82,6 +82,7 @@ impl DdlServiceImpl {
     }
 
     fn extract_replace_table_info(change: ReplaceTablePlan) -> ReplaceTableInfo {
+        let job_type = change.get_job_type().unwrap_or_default();
         let mut source = change.source;
         let mut fragment_graph = change.fragment_graph.unwrap();
         let mut table = change.table.unwrap();
@@ -89,20 +90,14 @@ impl DdlServiceImpl {
             table.optional_associated_source_id
         {
             source.as_mut().unwrap().id = source_id;
-            fill_table_stream_graph_info(
-                &mut source,
-                &mut table,
-                TableJobType::General,
-                &mut fragment_graph,
-            );
+            fill_table_stream_graph_info(&mut source, &mut table, job_type, &mut fragment_graph);
         }
         let table_col_index_mapping = change
             .table_col_index_mapping
             .as_ref()
             .map(ColIndexMapping::from_protobuf);
 
-        let stream_job = StreamingJob::Table(source, table, TableJobType::General);
-
+        let stream_job = StreamingJob::Table(source, table, job_type);
         ReplaceTableInfo {
             streaming_job: stream_job,
             fragment_graph,
@@ -740,7 +735,7 @@ impl DdlService for DdlServiceImpl {
         _request: Request<GetDdlProgressRequest>,
     ) -> Result<Response<GetDdlProgressResponse>, Status> {
         Ok(Response::new(GetDdlProgressResponse {
-            ddl_progress: self.ddl_controller.get_ddl_progress().await,
+            ddl_progress: self.ddl_controller.get_ddl_progress().await?,
         }))
     }
 
diff --git a/src/meta/service/src/hummock_service.rs b/src/meta/service/src/hummock_service.rs
index 1e46438cb8ddd..579c6ffda7de0 100644
--- a/src/meta/service/src/hummock_service.rs
+++ b/src/meta/service/src/hummock_service.rs
@@ -15,9 +15,11 @@
 use std::collections::{HashMap, HashSet};
 use std::time::Duration;
 
+use compact_task::PbTaskStatus;
 use futures::StreamExt;
 use itertools::Itertools;
 use risingwave_common::catalog::{TableId, SYS_CATALOG_START_ID};
+use risingwave_hummock_sdk::key_range::KeyRange;
 use risingwave_hummock_sdk::version::HummockVersionDelta;
 use risingwave_meta::manager::MetadataManager;
 use risingwave_pb::hummock::get_compaction_score_response::PickerInfo;
@@ -85,7 +87,7 @@ impl HummockManagerService for HummockServiceImpl {
         let current_version = self.hummock_manager.get_current_version().await;
         Ok(Response::new(GetCurrentVersionResponse {
             status: None,
-            current_version: Some(current_version.to_protobuf()),
+            current_version: Some(current_version.into()),
         }))
     }
 
@@ -101,7 +103,7 @@ impl HummockManagerService for HummockServiceImpl {
             ))
             .await?;
         Ok(Response::new(ReplayVersionDeltaResponse {
-            version: Some(version.to_protobuf()),
+            version: Some(version.into()),
             modified_compaction_groups: compaction_groups,
         }))
     }
@@ -123,7 +125,7 @@ impl HummockManagerService for HummockServiceImpl {
     ) -> Result<Response<DisableCommitEpochResponse>, Status> {
         let version = self.hummock_manager.disable_commit_epoch().await;
         Ok(Response::new(DisableCommitEpochResponse {
-            current_version: Some(version.to_protobuf()),
+            current_version: Some(version.into()),
         }))
     }
 
@@ -139,8 +141,8 @@ impl HummockManagerService for HummockServiceImpl {
         let resp = ListVersionDeltasResponse {
             version_deltas: Some(PbHummockVersionDeltas {
                 version_deltas: version_deltas
-                    .iter()
-                    .map(HummockVersionDelta::to_protobuf)
+                    .into_iter()
+                    .map(HummockVersionDelta::into)
                     .collect(),
             }),
         };
@@ -234,8 +236,12 @@ impl HummockManagerService for HummockServiceImpl {
 
         // rewrite the key_range
         match request.key_range {
-            Some(key_range) => {
-                option.key_range = key_range;
+            Some(pb_key_range) => {
+                option.key_range = KeyRange {
+                    left: pb_key_range.left.into(),
+                    right: pb_key_range.right.into(),
+                    right_exclusive: pb_key_range.right_exclusive,
+                };
             }
 
             None => {
@@ -320,9 +326,9 @@ impl HummockManagerService for HummockServiceImpl {
         &self,
         request: Request<TriggerFullGcRequest>,
     ) -> Result<Response<TriggerFullGcResponse>, Status> {
-        self.hummock_manager.start_full_gc(Duration::from_secs(
-            request.into_inner().sst_retention_time_sec,
-        ))?;
+        let req = request.into_inner();
+        self.hummock_manager
+            .start_full_gc(Duration::from_secs(req.sst_retention_time_sec), req.prefix)?;
         Ok(Response::new(TriggerFullGcResponse { status: None }))
     }
 
@@ -426,7 +432,7 @@ impl HummockManagerService for HummockServiceImpl {
         let req = request.into_inner();
         let version = self.hummock_manager.pin_version(req.context_id).await?;
         Ok(Response::new(PinVersionResponse {
-            pinned_version: Some(version.to_protobuf()),
+            pinned_version: Some(version.into()),
         }))
     }
 
@@ -464,7 +470,7 @@ impl HummockManagerService for HummockServiceImpl {
     ) -> Result<Response<RiseCtlGetCheckpointVersionResponse>, Status> {
         let checkpoint_version = self.hummock_manager.get_checkpoint_version().await;
         Ok(Response::new(RiseCtlGetCheckpointVersionResponse {
-            checkpoint_version: Some(checkpoint_version.to_protobuf()),
+            checkpoint_version: Some(checkpoint_version.into()),
         }))
     }
 
@@ -660,7 +666,10 @@ impl HummockManagerService for HummockServiceImpl {
         let request = request.into_inner();
         let ret = self
             .hummock_manager
-            .cancel_compact_task(request.task_id, request.task_status())
+            .cancel_compact_task(
+                request.task_id,
+                PbTaskStatus::try_from(request.task_status).unwrap(),
+            )
             .await?;
 
         let response = Response::new(CancelCompactTaskResponse { ret });
@@ -682,6 +691,17 @@ impl HummockManagerService for HummockServiceImpl {
             .await;
         Ok(Response::new(ListChangeLogEpochsResponse { epochs }))
     }
+
+    async fn get_version_by_epoch(
+        &self,
+        request: Request<GetVersionByEpochRequest>,
+    ) -> Result<Response<GetVersionByEpochResponse>, Status> {
+        let GetVersionByEpochRequest { epoch } = request.into_inner();
+        let version = self.hummock_manager.epoch_to_version(epoch).await?;
+        Ok(Response::new(GetVersionByEpochResponse {
+            version: Some(version.to_protobuf()),
+        }))
+    }
 }
 
 #[cfg(test)]
diff --git a/src/meta/service/src/lib.rs b/src/meta/service/src/lib.rs
index 80a83349f2cc7..9ab248802772e 100644
--- a/src/meta/service/src/lib.rs
+++ b/src/meta/service/src/lib.rs
@@ -14,7 +14,6 @@
 
 #![feature(lint_reasons)]
 #![feature(let_chains)]
-#![feature(lazy_cell)]
 #![feature(impl_trait_in_assoc_type)]
 #![cfg_attr(coverage, feature(coverage_attribute))]
 
diff --git a/src/meta/service/src/meta_member_service.rs b/src/meta/service/src/meta_member_service.rs
index b8f5d9ebf92c4..946337d248485 100644
--- a/src/meta/service/src/meta_member_service.rs
+++ b/src/meta/service/src/meta_member_service.rs
@@ -12,7 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use either::Either;
 use risingwave_common::util::addr::HostAddr;
 use risingwave_meta::rpc::ElectionClientRef;
 use risingwave_pb::common::HostAddress;
@@ -20,17 +19,14 @@ use risingwave_pb::meta::meta_member_service_server::MetaMemberService;
 use risingwave_pb::meta::{MembersRequest, MembersResponse, MetaMember};
 use tonic::{Request, Response, Status};
 
-use crate::AddressInfo;
 #[derive(Clone)]
 pub struct MetaMemberServiceImpl {
-    election_client_or_self: Either<ElectionClientRef, AddressInfo>,
+    election_client: ElectionClientRef,
 }
 
 impl MetaMemberServiceImpl {
-    pub fn new(election_client_or_self: Either<ElectionClientRef, AddressInfo>) -> Self {
-        MetaMemberServiceImpl {
-            election_client_or_self,
-        }
+    pub fn new(election_client: ElectionClientRef) -> Self {
+        MetaMemberServiceImpl { election_client }
     }
 }
 
@@ -41,39 +37,20 @@ impl MetaMemberService for MetaMemberServiceImpl {
         &self,
         _request: Request<MembersRequest>,
     ) -> Result<Response<MembersResponse>, Status> {
-        let members = match &self.election_client_or_self {
-            Either::Left(election_client) => {
-                let mut members = vec![];
-                for member in election_client.get_members().await? {
-                    let host_addr = member
-                        .id
-                        .parse::<HostAddr>()
-                        .map_err(|err| Status::from_error(err.into()))?;
-                    members.push(MetaMember {
-                        address: Some(HostAddress {
-                            host: host_addr.host,
-                            port: host_addr.port.into(),
-                        }),
-                        is_leader: member.is_leader,
-                    })
-                }
-
-                members
-            }
-            Either::Right(self_as_leader) => {
-                let host_addr = self_as_leader
-                    .advertise_addr
-                    .parse::<HostAddr>()
-                    .map_err(|err| Status::from_error(err.into()))?;
-                vec![MetaMember {
-                    address: Some(HostAddress {
-                        host: host_addr.host,
-                        port: host_addr.port.into(),
-                    }),
-                    is_leader: true,
-                }]
-            }
-        };
+        let mut members = vec![];
+        for member in self.election_client.get_members().await? {
+            let host_addr = member
+                .id
+                .parse::<HostAddr>()
+                .map_err(|err| Status::from_error(err.into()))?;
+            members.push(MetaMember {
+                address: Some(HostAddress {
+                    host: host_addr.host,
+                    port: host_addr.port.into(),
+                }),
+                is_leader: member.is_leader,
+            })
+        }
 
         Ok(Response::new(MembersResponse { members }))
     }
diff --git a/src/meta/service/src/notification_service.rs b/src/meta/service/src/notification_service.rs
index e4a8d298e0788..96ab254211d1c 100644
--- a/src/meta/service/src/notification_service.rs
+++ b/src/meta/service/src/notification_service.rs
@@ -12,12 +12,13 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use anyhow::Context;
+use anyhow::{anyhow, Context};
 use itertools::Itertools;
+use risingwave_common::secret::{LocalSecretManager, SecretEncryption};
 use risingwave_meta::manager::{MetadataManager, SessionParamsManagerImpl};
 use risingwave_meta::MetaResult;
 use risingwave_pb::backup_service::MetaBackupManifestId;
-use risingwave_pb::catalog::Table;
+use risingwave_pb::catalog::{Secret, Table};
 use risingwave_pb::common::worker_node::State::Running;
 use risingwave_pb::common::{WorkerNode, WorkerType};
 use risingwave_pb::hummock::WriteLimits;
@@ -47,20 +48,23 @@ pub struct NotificationServiceImpl {
 }
 
 impl NotificationServiceImpl {
-    pub fn new(
+    pub async fn new(
         env: MetaSrvEnv,
         metadata_manager: MetadataManager,
         hummock_manager: HummockManagerRef,
         backup_manager: BackupManagerRef,
         serving_vnode_mapping: ServingVnodeMappingRef,
-    ) -> Self {
-        Self {
+    ) -> MetaResult<Self> {
+        let service = Self {
             env,
             metadata_manager,
             hummock_manager,
             backup_manager,
             serving_vnode_mapping,
-        }
+        };
+        let (secrets, _catalog_version) = service.get_decrypted_secret_snapshot().await?;
+        LocalSecretManager::global().init_secrets(secrets);
+        Ok(service)
     }
 
     async fn get_catalog_snapshot(
@@ -142,25 +146,70 @@ impl NotificationServiceImpl {
         }
     }
 
+    /// Get decrypted secret snapshot
+    async fn get_decrypted_secret_snapshot(
+        &self,
+    ) -> MetaResult<(Vec<Secret>, NotificationVersion)> {
+        let secrets = match &self.metadata_manager {
+            MetadataManager::V1(mgr) => {
+                let catalog_guard = mgr.catalog_manager.get_catalog_core_guard().await;
+                catalog_guard.database.list_secrets()
+            }
+            MetadataManager::V2(mgr) => {
+                let catalog_guard = mgr.catalog_controller.get_inner_read_guard().await;
+                catalog_guard.list_secrets().await?
+            }
+        };
+        let notification_version = self.env.notification_manager().current_version().await;
+
+        let decrypted_secrets = self.decrypt_secrets(secrets)?;
+
+        Ok((decrypted_secrets, notification_version))
+    }
+
+    fn decrypt_secrets(&self, secrets: Vec<Secret>) -> MetaResult<Vec<Secret>> {
+        // Skip getting `secret_store_private_key` if there is no secret
+        if secrets.is_empty() {
+            return Ok(vec![]);
+        }
+        let secret_store_private_key = self
+            .env
+            .opts
+            .secret_store_private_key
+            .clone()
+            .ok_or_else(|| anyhow!("secret_store_private_key is not configured"))?;
+        let mut decrypted_secrets = Vec::with_capacity(secrets.len());
+        for mut secret in secrets {
+            let encrypted_secret = SecretEncryption::deserialize(secret.get_value())
+                .context(format!("failed to deserialize secret {}", secret.name))?;
+            let decrypted_secret = encrypted_secret
+                .decrypt(secret_store_private_key.as_slice())
+                .context(format!("failed to decrypt secret {}", secret.name))?;
+            secret.value = decrypted_secret;
+            decrypted_secrets.push(secret);
+        }
+        Ok(decrypted_secrets)
+    }
+
     async fn get_worker_slot_mapping_snapshot(
         &self,
     ) -> MetaResult<(Vec<FragmentWorkerSlotMapping>, NotificationVersion)> {
         match &self.metadata_manager {
             MetadataManager::V1(mgr) => {
                 let fragment_guard = mgr.fragment_manager.get_fragment_read_guard().await;
-                let parallel_unit_mappings =
+                let worker_slot_mappings =
                     fragment_guard.all_running_fragment_mappings().collect_vec();
                 let notification_version = self.env.notification_manager().current_version().await;
-                Ok((parallel_unit_mappings, notification_version))
+                Ok((worker_slot_mappings, notification_version))
             }
             MetadataManager::V2(mgr) => {
                 let fragment_guard = mgr.catalog_controller.get_inner_read_guard().await;
-                let parallel_unit_mappings = fragment_guard
+                let worker_slot_mappings = fragment_guard
                     .all_running_fragment_mappings()
                     .await?
                     .collect_vec();
                 let notification_version = self.env.notification_manager().current_version().await;
-                Ok((parallel_unit_mappings, notification_version))
+                Ok((worker_slot_mappings, notification_version))
             }
         }
     }
@@ -247,6 +296,9 @@ impl NotificationServiceImpl {
             catalog_version,
         ) = self.get_catalog_snapshot().await?;
 
+        // Use the plain text secret value for frontend. The secret value will be masked in frontend handle.
+        let decrypted_secrets = self.decrypt_secrets(secrets)?;
+
         let (streaming_worker_slot_mappings, streaming_worker_slot_mapping_version) =
             self.get_worker_slot_mapping_snapshot().await?;
         let serving_worker_slot_mappings = self.get_serving_vnode_mappings();
@@ -276,7 +328,7 @@ impl NotificationServiceImpl {
             subscriptions,
             functions,
             connections,
-            secrets,
+            secrets: decrypted_secrets,
             users,
             nodes,
             hummock_snapshot,
@@ -300,7 +352,7 @@ impl NotificationServiceImpl {
 
         Ok(MetaSnapshot {
             tables,
-            hummock_version: Some(hummock_version.to_protobuf()),
+            hummock_version: Some(hummock_version.into()),
             version: Some(SnapshotVersion {
                 catalog_version,
                 ..Default::default()
@@ -315,8 +367,16 @@ impl NotificationServiceImpl {
         })
     }
 
-    fn compute_subscribe(&self) -> MetaSnapshot {
-        MetaSnapshot::default()
+    async fn compute_subscribe(&self) -> MetaResult<MetaSnapshot> {
+        let (secrets, catalog_version) = self.get_decrypted_secret_snapshot().await?;
+        Ok(MetaSnapshot {
+            secrets,
+            version: Some(SnapshotVersion {
+                catalog_version,
+                ..Default::default()
+            }),
+            ..Default::default()
+        })
     }
 }
 
@@ -355,7 +415,7 @@ impl NotificationService for NotificationServiceImpl {
                     .await?;
                 self.hummock_subscribe().await?
             }
-            SubscribeType::Compute => self.compute_subscribe(),
+            SubscribeType::Compute => self.compute_subscribe().await?,
             SubscribeType::Unspecified => unreachable!(),
         };
 
diff --git a/src/meta/service/src/scale_service.rs b/src/meta/service/src/scale_service.rs
index 273fdbca9bc07..56040bb5c973f 100644
--- a/src/meta/service/src/scale_service.rs
+++ b/src/meta/service/src/scale_service.rs
@@ -14,32 +14,32 @@
 
 use std::collections::HashMap;
 
-use risingwave_common::catalog;
+use risingwave_common::catalog::TableId;
 use risingwave_meta::manager::MetadataManager;
 use risingwave_meta::model::TableParallelism;
-use risingwave_meta::stream::{ScaleControllerRef, TableRevision};
+use risingwave_meta::stream::{
+    RescheduleOptions, ScaleControllerRef, TableRevision, WorkerReschedule,
+};
 use risingwave_meta_model_v2::FragmentId;
 use risingwave_pb::common::WorkerType;
 use risingwave_pb::meta::scale_service_server::ScaleService;
 use risingwave_pb::meta::{
-    GetClusterInfoRequest, GetClusterInfoResponse, GetReschedulePlanRequest,
-    GetReschedulePlanResponse, Reschedule, RescheduleRequest, RescheduleResponse,
+    GetClusterInfoRequest, GetClusterInfoResponse, GetServerlessStreamingJobsStatusRequest,
+    GetServerlessStreamingJobsStatusResponse, PbWorkerReschedule, RescheduleRequest,
+    RescheduleResponse, UpdateStreamingJobNodeLabelsRequest, UpdateStreamingJobNodeLabelsResponse,
 };
 use risingwave_pb::source::{ConnectorSplit, ConnectorSplits};
 use tonic::{Request, Response, Status};
 
 use crate::barrier::BarrierManagerRef;
 use crate::model::MetadataModel;
-use crate::stream::{
-    GlobalStreamManagerRef, ParallelUnitReschedule, RescheduleOptions, SourceManagerRef,
-};
+use crate::stream::{GlobalStreamManagerRef, SourceManagerRef};
 
 pub struct ScaleServiceImpl {
     metadata_manager: MetadataManager,
     source_manager: SourceManagerRef,
     stream_manager: GlobalStreamManagerRef,
     barrier_manager: BarrierManagerRef,
-    scale_controller: ScaleControllerRef,
 }
 
 impl ScaleServiceImpl {
@@ -48,14 +48,13 @@ impl ScaleServiceImpl {
         source_manager: SourceManagerRef,
         stream_manager: GlobalStreamManagerRef,
         barrier_manager: BarrierManagerRef,
-        scale_controller: ScaleControllerRef,
+        _scale_controller: ScaleControllerRef,
     ) -> Self {
         Self {
             metadata_manager,
             source_manager,
             stream_manager,
             barrier_manager,
-            scale_controller,
         }
     }
 
@@ -137,7 +136,7 @@ impl ScaleService for ScaleServiceImpl {
         self.barrier_manager.check_status_running()?;
 
         let RescheduleRequest {
-            reschedules,
+            worker_reschedules,
             revision,
             resolve_no_shuffle_upstream,
         } = request.into_inner();
@@ -162,7 +161,7 @@ impl ScaleService for ScaleServiceImpl {
                     for (table_id, table) in guard.table_fragments() {
                         if table
                             .fragment_ids()
-                            .any(|fragment_id| reschedules.contains_key(&fragment_id))
+                            .any(|fragment_id| worker_reschedules.contains_key(&fragment_id))
                         {
                             table_parallelisms.insert(*table_id, TableParallelism::Custom);
                         }
@@ -174,13 +173,16 @@ impl ScaleService for ScaleServiceImpl {
                     let streaming_job_ids = mgr
                         .catalog_controller
                         .get_fragment_job_id(
-                            reschedules.keys().map(|id| *id as FragmentId).collect(),
+                            worker_reschedules
+                                .keys()
+                                .map(|id| *id as FragmentId)
+                                .collect(),
                         )
                         .await?;
 
                     streaming_job_ids
                         .into_iter()
-                        .map(|id| (catalog::TableId::new(id as _), TableParallelism::Custom))
+                        .map(|id| (TableId::new(id as _), TableParallelism::Custom))
                         .collect()
                 }
             }
@@ -188,22 +190,17 @@ impl ScaleService for ScaleServiceImpl {
 
         self.stream_manager
             .reschedule_actors(
-                reschedules
+                worker_reschedules
                     .into_iter()
                     .map(|(fragment_id, reschedule)| {
-                        let Reschedule {
-                            added_parallel_units,
-                            removed_parallel_units,
-                        } = reschedule;
-
-                        let added_parallel_units = added_parallel_units.into_iter().collect();
-                        let removed_parallel_units = removed_parallel_units.into_iter().collect();
-
+                        let PbWorkerReschedule { worker_actor_diff } = reschedule;
                         (
                             fragment_id,
-                            ParallelUnitReschedule {
-                                added_parallel_units,
-                                removed_parallel_units,
+                            WorkerReschedule {
+                                worker_actor_diff: worker_actor_diff
+                                    .into_iter()
+                                    .map(|(worker_id, diff)| (worker_id as _, diff as _))
+                                    .collect(),
                             },
                         )
                     })
@@ -224,61 +221,17 @@ impl ScaleService for ScaleServiceImpl {
         }))
     }
 
-    #[cfg_attr(coverage, coverage(off))]
-    async fn get_reschedule_plan(
+    async fn update_streaming_job_node_labels(
         &self,
-        request: Request<GetReschedulePlanRequest>,
-    ) -> Result<Response<GetReschedulePlanResponse>, Status> {
-        self.barrier_manager.check_status_running()?;
-
-        let req = request.into_inner();
-
-        let _reschedule_job_lock = self.stream_manager.reschedule_lock_read_guard().await;
-
-        let current_revision = self.get_revision().await;
-
-        if req.revision != current_revision.inner() {
-            return Ok(Response::new(GetReschedulePlanResponse {
-                success: false,
-                revision: current_revision.inner(),
-                reschedules: Default::default(),
-            }));
-        }
-
-        let policy = req
-            .policy
-            .ok_or_else(|| Status::invalid_argument("policy is required"))?;
-
-        let scale_controller = &self.scale_controller;
-
-        let plan = scale_controller.get_reschedule_plan(policy).await?;
-
-        let next_revision = self.get_revision().await;
-
-        // generate reschedule plan will not change the revision
-        assert_eq!(current_revision, next_revision);
+        _request: Request<UpdateStreamingJobNodeLabelsRequest>,
+    ) -> Result<Response<UpdateStreamingJobNodeLabelsResponse>, Status> {
+        todo!()
+    }
 
-        Ok(Response::new(GetReschedulePlanResponse {
-            success: true,
-            revision: next_revision.into(),
-            reschedules: plan
-                .into_iter()
-                .map(|(fragment_id, reschedule)| {
-                    (
-                        fragment_id,
-                        Reschedule {
-                            added_parallel_units: reschedule
-                                .added_parallel_units
-                                .into_iter()
-                                .collect(),
-                            removed_parallel_units: reschedule
-                                .removed_parallel_units
-                                .into_iter()
-                                .collect(),
-                        },
-                    )
-                })
-                .collect(),
-        }))
+    async fn get_serverless_streaming_jobs_status(
+        &self,
+        _request: Request<GetServerlessStreamingJobsStatusRequest>,
+    ) -> Result<Response<GetServerlessStreamingJobsStatusResponse>, Status> {
+        todo!()
     }
 }
diff --git a/src/meta/service/src/stream_service.rs b/src/meta/service/src/stream_service.rs
index be520132b167f..d50a088972eeb 100644
--- a/src/meta/service/src/stream_service.rs
+++ b/src/meta/service/src/stream_service.rs
@@ -374,7 +374,7 @@ impl StreamManagerService for StreamServiceImpl {
                                 actor_id,
                                 fragment_id: actor_to_fragment[&actor_id],
                                 state: status.state,
-                                parallel_unit_id: status.parallel_unit.as_ref().unwrap().id,
+                                worker_id: status.worker_id(),
                             }
                         })
                     })
@@ -388,7 +388,7 @@ impl StreamManagerService for StreamServiceImpl {
                         actor_id: actor_location.actor_id as _,
                         fragment_id: actor_location.fragment_id as _,
                         state: PbActorState::from(actor_location.status) as _,
-                        parallel_unit_id: actor_location.parallel_unit_id as _,
+                        worker_id: actor_location.worker_id as _,
                     })
                     .collect_vec()
             }
diff --git a/src/meta/src/backup_restore/meta_snapshot_builder.rs b/src/meta/src/backup_restore/meta_snapshot_builder.rs
index bb8a1eb919fdc..6696bd534b78a 100644
--- a/src/meta/src/backup_restore/meta_snapshot_builder.rs
+++ b/src/meta/src/backup_restore/meta_snapshot_builder.rs
@@ -21,7 +21,7 @@ use risingwave_backup::meta_snapshot_v1::{ClusterMetadata, MetaSnapshotV1};
 use risingwave_backup::MetaSnapshotId;
 use risingwave_hummock_sdk::version::{HummockVersion, HummockVersionDelta};
 use risingwave_pb::catalog::{
-    Connection, Database, Function, Index, Schema, Sink, Source, Subscription, Table, View,
+    Connection, Database, Function, Index, Schema, Secret, Sink, Source, Subscription, Table, View,
 };
 use risingwave_pb::hummock::HummockVersionStats;
 use risingwave_pb::meta::SystemParams;
@@ -124,6 +124,7 @@ impl<S: MetaStore> MetaSnapshotV1Builder<S> {
             .ok_or_else(|| anyhow!("cluster id not found in meta store"))?
             .into();
         let subscription = Subscription::list_at_snapshot::<S>(&meta_store_snapshot).await?;
+        let secret = Secret::list_at_snapshot::<S>(&meta_store_snapshot).await?;
 
         self.snapshot.metadata = ClusterMetadata {
             default_cf,
@@ -144,6 +145,7 @@ impl<S: MetaStore> MetaSnapshotV1Builder<S> {
             system_param,
             cluster_id,
             subscription,
+            secret,
         };
         Ok(())
     }
diff --git a/src/meta/src/backup_restore/restore.rs b/src/meta/src/backup_restore/restore.rs
index 6165ba331d857..bc22c74c39442 100644
--- a/src/meta/src/backup_restore/restore.rs
+++ b/src/meta/src/backup_restore/restore.rs
@@ -79,6 +79,12 @@ pub struct RestoreOpts {
     /// Print the target snapshot, but won't restore to meta store.
     #[clap(long)]
     pub dry_run: bool,
+    /// The read timeout for object store
+    #[clap(long, default_value_t = 600000)]
+    pub read_attempt_timeout_ms: u64,
+    /// The maximum number of read retry attempts for the object store.
+    #[clap(long, default_value_t = 3)]
+    pub read_retry_attempts: u64,
 }
 
 async fn restore_hummock_version(
@@ -97,7 +103,7 @@ async fn restore_hummock_version(
     );
     let checkpoint_path = version_checkpoint_path(hummock_storage_directory);
     let checkpoint = PbHummockVersionCheckpoint {
-        version: Some(hummock_version.to_protobuf()),
+        version: Some(hummock_version.into()),
         // Ignore stale objects. Full GC will clear them.
         stale_objects: Default::default(),
     };
@@ -251,6 +257,8 @@ mod tests {
             hummock_storage_url: "memory".to_string(),
             hummock_storage_directory: "".to_string(),
             dry_run: false,
+            read_attempt_timeout_ms: 60000,
+            read_retry_attempts: 3,
         }
     }
 
diff --git a/src/meta/src/backup_restore/restore_impl/v1.rs b/src/meta/src/backup_restore/restore_impl/v1.rs
index 303e6e6ad5f52..0ceee8daa40ba 100644
--- a/src/meta/src/backup_restore/restore_impl/v1.rs
+++ b/src/meta/src/backup_restore/restore_impl/v1.rs
@@ -43,7 +43,7 @@ impl Loader<ClusterMetadata> for LoaderV1 {
         let backup_store = &self.backup_store;
         let snapshot_list = &backup_store.manifest().snapshot_metadata;
         let mut target_snapshot: MetaSnapshotV1 = backup_store.get(target_id).await?;
-        tracing::info!(
+        tracing::debug!(
             "snapshot {} before rewrite:\n{}",
             target_id,
             target_snapshot
@@ -78,11 +78,11 @@ impl Loader<ClusterMetadata> for LoaderV1 {
             }
             target_snapshot.metadata.default_cf = newest_snapshot.metadata.default_cf;
             tracing::info!(
-                "snapshot {} after rewrite by snapshot {}:\n{}",
+                "snapshot {} is rewritten by snapshot {}:\n",
                 target_id,
                 newest_id,
-                target_snapshot,
             );
+            tracing::debug!("{target_snapshot}",);
         }
         Ok(target_snapshot)
     }
@@ -199,5 +199,6 @@ async fn restore_metadata<S: MetaStore>(
     restore_system_param_model(&meta_store, &[snapshot.metadata.system_param]).await?;
     restore_cluster_id(&meta_store, snapshot.metadata.cluster_id.into()).await?;
     restore_metadata_model(&meta_store, &snapshot.metadata.subscription).await?;
+    restore_metadata_model(&meta_store, &snapshot.metadata.secret).await?;
     Ok(())
 }
diff --git a/src/meta/src/backup_restore/restore_impl/v2.rs b/src/meta/src/backup_restore/restore_impl/v2.rs
index a431b455063bb..a887293e0c8ef 100644
--- a/src/meta/src/backup_restore/restore_impl/v2.rs
+++ b/src/meta/src/backup_restore/restore_impl/v2.rs
@@ -37,7 +37,7 @@ impl Loader<MetadataV2> for LoaderV2 {
     async fn load(&self, target_id: MetaSnapshotId) -> BackupResult<MetaSnapshot<MetadataV2>> {
         let snapshot_list = &self.backup_store.manifest().snapshot_metadata;
         let mut target_snapshot: MetaSnapshotV2 = self.backup_store.get(target_id).await?;
-        tracing::info!(
+        tracing::debug!(
             "snapshot {} before rewrite:\n{}",
             target_id,
             target_snapshot
@@ -73,11 +73,11 @@ impl Loader<MetadataV2> for LoaderV2 {
             }
             target_snapshot.metadata.hummock_sequences = newest_snapshot.metadata.hummock_sequences;
             tracing::info!(
-                "snapshot {} after rewrite by snapshot {}:\n{}",
+                "snapshot {} is rewritten by snapshot {}:\n",
                 target_id,
                 newest_id,
-                target_snapshot,
             );
+            tracing::debug!("{target_snapshot}");
         }
         Ok(target_snapshot)
     }
diff --git a/src/meta/src/backup_restore/utils.rs b/src/meta/src/backup_restore/utils.rs
index 6f6c1dd09ece0..4c4c9a336b449 100644
--- a/src/meta/src/backup_restore/utils.rs
+++ b/src/meta/src/backup_restore/utils.rs
@@ -120,11 +120,15 @@ pub async fn get_meta_store(opts: RestoreOpts) -> BackupResult<MetaStoreBackendI
 }
 
 pub async fn get_backup_store(opts: RestoreOpts) -> BackupResult<MetaSnapshotStorageRef> {
+    let mut config = ObjectStoreConfig::default();
+    config.retry.read_attempt_timeout_ms = opts.read_attempt_timeout_ms;
+    config.retry.read_retry_attempts = opts.read_retry_attempts as usize;
+
     let object_store = build_remote_object_store(
         &opts.backup_storage_url,
         Arc::new(ObjectStoreMetrics::unused()),
         "Meta Backup",
-        Arc::new(ObjectStoreConfig::default()),
+        Arc::new(config),
     )
     .await;
     let backup_store =
diff --git a/src/meta/src/barrier/command.rs b/src/meta/src/barrier/command.rs
index aae2f2f42a2f5..234c4b36c1c5e 100644
--- a/src/meta/src/barrier/command.rs
+++ b/src/meta/src/barrier/command.rs
@@ -13,18 +13,19 @@
 // limitations under the License.
 
 use std::collections::{HashMap, HashSet};
+use std::fmt::Formatter;
 use std::sync::Arc;
 
 use futures::future::try_join_all;
 use itertools::Itertools;
-use risingwave_common::buffer::Bitmap;
+use risingwave_common::bitmap::Bitmap;
 use risingwave_common::catalog::TableId;
 use risingwave_common::hash::ActorMapping;
 use risingwave_common::types::Timestamptz;
 use risingwave_common::util::epoch::Epoch;
 use risingwave_connector::source::SplitImpl;
 use risingwave_hummock_sdk::HummockEpoch;
-use risingwave_pb::catalog::CreateType;
+use risingwave_pb::catalog::{CreateType, Table};
 use risingwave_pb::meta::table_fragments::PbActorStatus;
 use risingwave_pb::meta::PausedReason;
 use risingwave_pb::source::{ConnectorSplit, ConnectorSplits};
@@ -33,9 +34,9 @@ use risingwave_pb::stream_plan::barrier_mutation::Mutation;
 use risingwave_pb::stream_plan::throttle_mutation::RateLimit;
 use risingwave_pb::stream_plan::update_mutation::*;
 use risingwave_pb::stream_plan::{
-    AddMutation, BarrierMutation, CombinedMutation, CreateSubscriptionMutation, Dispatcher,
-    Dispatchers, DropSubscriptionMutation, PauseMutation, ResumeMutation,
-    SourceChangeSplitMutation, StopMutation, StreamActor, ThrottleMutation, UpdateMutation,
+    AddMutation, BarrierMutation, CombinedMutation, Dispatcher, Dispatchers,
+    DropSubscriptionsMutation, PauseMutation, ResumeMutation, SourceChangeSplitMutation,
+    StopMutation, StreamActor, SubscriptionUpstreamInfo, ThrottleMutation, UpdateMutation,
 };
 use risingwave_pb::stream_service::WaitEpochCommitRequest;
 use thiserror_ext::AsReport;
@@ -44,7 +45,7 @@ use tracing::warn;
 use super::info::{CommandActorChanges, CommandFragmentChanges, InflightActorInfo};
 use super::trace::TracedEpoch;
 use crate::barrier::GlobalBarrierManagerContext;
-use crate::manager::{DdlType, MetadataManager, WorkerId};
+use crate::manager::{DdlType, InflightFragmentInfo, MetadataManager, StreamingJob, WorkerId};
 use crate::model::{ActorId, DispatcherId, FragmentId, TableFragments, TableParallelism};
 use crate::stream::{build_actor_connector_splits, SplitAssignment, ThrottleConfig};
 use crate::MetaResult;
@@ -73,7 +74,8 @@ pub struct Reschedule {
     /// The downstream fragments of this fragment.
     pub downstream_fragment_ids: Vec<FragmentId>,
 
-    /// Reassigned splits for source actors
+    /// Reassigned splits for source actors.
+    /// It becomes the `actor_splits` in [`UpdateMutation`].
     pub actor_splits: HashMap<ActorId, Vec<SplitImpl>>,
 
     /// Whether this fragment is injectable. The injectable means whether the fragment contains
@@ -97,14 +99,18 @@ pub struct ReplaceTablePlan {
     /// Note that there's no `SourceBackfillExecutor` involved for table with connector, so we don't need to worry about
     /// `backfill_splits`.
     pub init_split_assignment: SplitAssignment,
+    /// The `StreamingJob` info of the table to be replaced. Must be `StreamingJob::Table`
+    pub streaming_job: StreamingJob,
+    /// The temporary dummy table fragments id of new table fragment
+    pub dummy_id: u32,
 }
 
 impl ReplaceTablePlan {
     fn actor_changes(&self) -> CommandActorChanges {
         let mut fragment_changes = HashMap::new();
         for fragment in self.new_table_fragments.fragments.values() {
-            let fragment_change = CommandFragmentChanges::NewFragment {
-                new_actors: fragment
+            let fragment_change = CommandFragmentChanges::NewFragment(InflightFragmentInfo {
+                actors: fragment
                     .actors
                     .iter()
                     .map(|actor| {
@@ -114,19 +120,17 @@ impl ReplaceTablePlan {
                                 .actor_status
                                 .get(&actor.actor_id)
                                 .expect("should exist")
-                                .get_parallel_unit()
-                                .expect("should set")
-                                .worker_node_id,
+                                .worker_id(),
                         )
                     })
                     .collect(),
-                table_ids: fragment
+                state_table_ids: fragment
                     .state_table_ids
                     .iter()
                     .map(|table_id| TableId::new(*table_id))
                     .collect(),
                 is_injectable: TableFragments::is_injectable(fragment.fragment_type_mask),
-            };
+            });
             assert!(fragment_changes
                 .insert(fragment.fragment_id, fragment_change)
                 .is_none());
@@ -140,6 +144,52 @@ impl ReplaceTablePlan {
     }
 }
 
+#[derive(Debug, Clone)]
+pub struct CreateStreamingJobCommandInfo {
+    pub table_fragments: TableFragments,
+    /// Refer to the doc on [`MetadataManager::get_upstream_root_fragments`] for the meaning of "root".
+    pub upstream_root_actors: HashMap<TableId, Vec<ActorId>>,
+    pub dispatchers: HashMap<ActorId, Vec<Dispatcher>>,
+    pub init_split_assignment: SplitAssignment,
+    pub definition: String,
+    pub ddl_type: DdlType,
+    pub create_type: CreateType,
+    pub streaming_job: StreamingJob,
+    pub internal_tables: Vec<Table>,
+}
+
+impl CreateStreamingJobCommandInfo {
+    fn new_fragment_info(&self) -> impl Iterator<Item = (FragmentId, InflightFragmentInfo)> + '_ {
+        self.table_fragments.fragments.values().map(|fragment| {
+            (
+                fragment.fragment_id,
+                InflightFragmentInfo {
+                    actors: fragment
+                        .actors
+                        .iter()
+                        .map(|actor| {
+                            (
+                                actor.actor_id,
+                                self.table_fragments
+                                    .actor_status
+                                    .get(&actor.actor_id)
+                                    .expect("should exist")
+                                    .worker_id(),
+                            )
+                        })
+                        .collect(),
+                    state_table_ids: fragment
+                        .state_table_ids
+                        .iter()
+                        .map(|table_id| TableId::new(*table_id))
+                        .collect(),
+                    is_injectable: TableFragments::is_injectable(fragment.fragment_type_mask),
+                },
+            )
+        })
+    }
+}
+
 /// [`Command`] is the input of [`crate::barrier::GlobalBarrierManager`]. For different commands,
 /// it will build different barriers to send, and may do different stuffs after the barrier is
 /// collected.
@@ -183,14 +233,7 @@ pub enum Command {
     /// for a while** until the `finish` channel is signaled, then the state of `TableFragments`
     /// will be set to `Created`.
     CreateStreamingJob {
-        table_fragments: TableFragments,
-        /// Refer to the doc on [`MetadataManager::get_upstream_root_fragments`] for the meaning of "root".
-        upstream_root_actors: HashMap<TableId, Vec<ActorId>>,
-        dispatchers: HashMap<ActorId, Vec<Dispatcher>>,
-        init_split_assignment: SplitAssignment,
-        definition: String,
-        ddl_type: DdlType,
-        create_type: CreateType,
+        info: CreateStreamingJobCommandInfo,
         /// This is for create SINK into table.
         replace_table: Option<ReplaceTablePlan>,
     },
@@ -236,7 +279,7 @@ pub enum Command {
         retention_second: u64,
     },
 
-    /// `DropSubscription` command generates a `DropSubscriptionMutation` to notify
+    /// `DropSubscription` command generates a `DropSubscriptionsMutation` to notify
     /// materialize executor to stop storing old value when there is no
     /// subscription depending on it.
     DropSubscription {
@@ -273,43 +316,13 @@ impl Command {
                     .collect(),
             }),
             Command::CreateStreamingJob {
-                table_fragments,
+                info,
                 replace_table,
-                ..
             } => {
-                let fragment_changes = table_fragments
-                    .fragments
-                    .values()
-                    .map(|fragment| {
-                        (
-                            fragment.fragment_id,
-                            CommandFragmentChanges::NewFragment {
-                                new_actors: fragment
-                                    .actors
-                                    .iter()
-                                    .map(|actor| {
-                                        (
-                                            actor.actor_id,
-                                            table_fragments
-                                                .actor_status
-                                                .get(&actor.actor_id)
-                                                .expect("should exist")
-                                                .get_parallel_unit()
-                                                .expect("should set")
-                                                .worker_node_id,
-                                        )
-                                    })
-                                    .collect(),
-                                table_ids: fragment
-                                    .state_table_ids
-                                    .iter()
-                                    .map(|table_id| TableId::new(*table_id))
-                                    .collect(),
-                                is_injectable: TableFragments::is_injectable(
-                                    fragment.fragment_type_mask,
-                                ),
-                            },
-                        )
+                let fragment_changes = info
+                    .new_fragment_info()
+                    .map(|(fragment_id, info)| {
+                        (fragment_id, CommandFragmentChanges::NewFragment(info))
                     })
                     .collect();
                 let mut changes = CommandActorChanges { fragment_changes };
@@ -431,6 +444,17 @@ pub struct CommandContext {
     pub _span: tracing::Span,
 }
 
+impl std::fmt::Debug for CommandContext {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("CommandContext")
+            .field("prev_epoch", &self.prev_epoch.value().0)
+            .field("curr_epoch", &self.curr_epoch.value().0)
+            .field("kind", &self.kind)
+            .field("command", &self.command)
+            .finish()
+    }
+}
+
 impl CommandContext {
     #[allow(clippy::too_many_arguments)]
     pub(super) fn new(
@@ -454,10 +478,6 @@ impl CommandContext {
             _span: span,
         }
     }
-
-    pub fn metadata_manager(&self) -> &MetadataManager {
-        &self.barrier_manager_context.metadata_manager
-    }
 }
 
 impl CommandContext {
@@ -515,11 +535,14 @@ impl CommandContext {
                 })),
 
                 Command::CreateStreamingJob {
-                    table_fragments,
-                    dispatchers,
-                    init_split_assignment: split_assignment,
+                    info:
+                        CreateStreamingJobCommandInfo {
+                            table_fragments,
+                            dispatchers,
+                            init_split_assignment: split_assignment,
+                            ..
+                        },
                     replace_table,
-                    ..
                 } => {
                     let actor_dispatchers = dispatchers
                         .iter()
@@ -543,6 +566,7 @@ impl CommandContext {
                         actor_splits,
                         // If the cluster is already paused, the new actors should be paused too.
                         pause: self.current_paused_reason.is_some(),
+                        subscriptions_to_add: Default::default(),
                     }));
 
                     if let Some(ReplaceTablePlan {
@@ -551,6 +575,7 @@ impl CommandContext {
                         merge_updates,
                         dispatchers,
                         init_split_assignment,
+                        ..
                     }) = replace_table
                     {
                         // TODO: support in v2.
@@ -736,16 +761,24 @@ impl CommandContext {
                     upstream_mv_table_id,
                     subscription_id,
                     ..
-                } => Some(Mutation::CreateSubscription(CreateSubscriptionMutation {
-                    upstream_mv_table_id: upstream_mv_table_id.table_id,
-                    subscription_id: *subscription_id,
+                } => Some(Mutation::Add(AddMutation {
+                    actor_dispatchers: Default::default(),
+                    added_actors: vec![],
+                    actor_splits: Default::default(),
+                    pause: false,
+                    subscriptions_to_add: vec![SubscriptionUpstreamInfo {
+                        upstream_mv_table_id: upstream_mv_table_id.table_id,
+                        subscriber_id: *subscription_id,
+                    }],
                 })),
                 Command::DropSubscription {
                     upstream_mv_table_id,
                     subscription_id,
-                } => Some(Mutation::DropSubscription(DropSubscriptionMutation {
-                    upstream_mv_table_id: upstream_mv_table_id.table_id,
-                    subscription_id: *subscription_id,
+                } => Some(Mutation::DropSubscriptions(DropSubscriptionsMutation {
+                    info: vec![SubscriptionUpstreamInfo {
+                        subscriber_id: *subscription_id,
+                        upstream_mv_table_id: upstream_mv_table_id.table_id,
+                    }],
                 })),
             };
 
@@ -810,39 +843,19 @@ impl CommandContext {
             _ => self.current_paused_reason,
         }
     }
+}
 
-    /// For `CreateStreamingJob`, returns the actors of the `StreamScan`, and `StreamValue` nodes. For other commands,
-    /// returns an empty set.
-    pub fn actors_to_track(&self) -> HashSet<ActorId> {
-        match &self.command {
-            Command::CreateStreamingJob {
-                table_fragments, ..
-            } => table_fragments
-                .tracking_progress_actor_ids()
-                .into_iter()
-                .collect(),
-            _ => Default::default(),
-        }
-    }
-
+impl Command {
     /// For `CancelStreamingJob`, returns the table id of the target table.
     pub fn table_to_cancel(&self) -> Option<TableId> {
-        match &self.command {
+        match self {
             Command::CancelStreamingJob(table_fragments) => Some(table_fragments.table_id()),
             _ => None,
         }
     }
+}
 
-    /// For `CreateStreamingJob`, returns the table id of the target table.
-    pub fn table_to_create(&self) -> Option<TableId> {
-        match &self.command {
-            Command::CreateStreamingJob {
-                table_fragments, ..
-            } => Some(table_fragments.table_id()),
-            _ => None,
-        }
-    }
-
+impl CommandContext {
     /// Clean up actors in CNs if needed, used by drop, cancel and reschedule commands.
     async fn clean_up(&self, actors: Vec<ActorId>) -> MetaResult<()> {
         self.barrier_manager_context
@@ -985,14 +998,16 @@ impl CommandContext {
             }
 
             Command::CreateStreamingJob {
-                table_fragments,
-                dispatchers,
-                upstream_root_actors,
-                init_split_assignment,
-                definition: _,
+                info,
                 replace_table,
-                ..
             } => {
+                let CreateStreamingJobCommandInfo {
+                    table_fragments,
+                    dispatchers,
+                    upstream_root_actors,
+                    init_split_assignment,
+                    ..
+                } = info;
                 match &self.barrier_manager_context.metadata_manager {
                     MetadataManager::V1(mgr) => {
                         let mut dependent_table_actors =
@@ -1019,6 +1034,7 @@ impl CommandContext {
                             merge_updates,
                             dispatchers,
                             init_split_assignment,
+                            ..
                         }) = replace_table
                         {
                             self.clean_up(old_table_fragments.actor_ids()).await?;
@@ -1104,6 +1120,7 @@ impl CommandContext {
                 merge_updates,
                 dispatchers,
                 init_split_assignment,
+                ..
             }) => {
                 self.clean_up(old_table_fragments.actor_ids()).await?;
 
diff --git a/src/meta/src/barrier/info.rs b/src/meta/src/barrier/info.rs
index f6617b9ceef47..44194c7f9eb30 100644
--- a/src/meta/src/barrier/info.rs
+++ b/src/meta/src/barrier/info.rs
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::{HashMap, HashSet};
+use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
 
 use risingwave_common::catalog::TableId;
 use risingwave_pb::common::PbWorkerNode;
@@ -24,11 +24,7 @@ use crate::model::{ActorId, FragmentId};
 
 #[derive(Debug, Clone)]
 pub(crate) enum CommandFragmentChanges {
-    NewFragment {
-        new_actors: HashMap<ActorId, WorkerId>,
-        table_ids: HashSet<TableId>,
-        is_injectable: bool,
-    },
+    NewFragment(InflightFragmentInfo),
     Reschedule {
         new_actors: HashMap<ActorId, WorkerId>,
         to_remove: HashSet<ActorId>,
@@ -62,9 +58,6 @@ pub struct InflightActorInfo {
     /// `node_id` => actors
     pub actor_map: HashMap<WorkerId, HashSet<ActorId>>,
 
-    /// `node_id` => barrier inject actors
-    pub actor_map_to_send: HashMap<WorkerId, HashSet<ActorId>>,
-
     /// `actor_id` => `WorkerId`
     pub actor_location_map: HashMap<ActorId, WorkerId>,
 
@@ -93,20 +86,6 @@ impl InflightActorInfo {
             map
         };
 
-        let actor_map_to_send = {
-            let mut map: HashMap<_, HashSet<_>> = HashMap::new();
-            for info in actor_infos
-                .fragment_infos
-                .values()
-                .filter(|info| info.is_injectable)
-            {
-                for (actor_id, worker_id) in &info.actors {
-                    map.entry(*worker_id).or_default().insert(*actor_id);
-                }
-            }
-            map
-        };
-
         let actor_location_map = actor_infos
             .fragment_infos
             .values()
@@ -121,7 +100,6 @@ impl InflightActorInfo {
         Self {
             node_map,
             actor_map,
-            actor_map_to_send,
             actor_location_map,
             mv_depended_subscriptions,
             fragment_infos: actor_infos.fragment_infos,
@@ -134,11 +112,26 @@ impl InflightActorInfo {
             .into_iter()
             .map(|node| (node.id, node))
             .collect::<HashMap<_, _>>();
-        for (actor_id, location) in &self.actor_location_map {
-            if !new_node_map.contains_key(location) {
-                warn!(actor_id, location, node = ?self.node_map.get(location), "node with running actors is deleted");
+
+        let mut deleted_actors = BTreeMap::new();
+        for (&actor_id, &location) in &self.actor_location_map {
+            if !new_node_map.contains_key(&location) {
+                deleted_actors
+                    .entry(location)
+                    .or_insert_with(BTreeSet::new)
+                    .insert(actor_id);
             }
         }
+        for (node_id, actors) in deleted_actors {
+            let node = self.node_map.get(&node_id);
+            warn!(
+                node_id,
+                ?node,
+                ?actors,
+                "node with running actors is deleted"
+            );
+        }
+
         self.node_map = new_node_map;
     }
 
@@ -149,27 +142,11 @@ impl InflightActorInfo {
             let mut to_add = HashMap::new();
             for (fragment_id, change) in fragment_changes {
                 match change {
-                    CommandFragmentChanges::NewFragment {
-                        new_actors,
-                        table_ids,
-                        is_injectable,
-                    } => {
-                        for (actor_id, node_id) in &new_actors {
-                            assert!(to_add
-                                .insert(*actor_id, (*node_id, is_injectable))
-                                .is_none());
+                    CommandFragmentChanges::NewFragment(info) => {
+                        for (actor_id, node_id) in &info.actors {
+                            assert!(to_add.insert(*actor_id, *node_id).is_none());
                         }
-                        assert!(self
-                            .fragment_infos
-                            .insert(
-                                fragment_id,
-                                InflightFragmentInfo {
-                                    actors: new_actors,
-                                    state_table_ids: table_ids,
-                                    is_injectable,
-                                }
-                            )
-                            .is_none());
+                        assert!(self.fragment_infos.insert(fragment_id, info).is_none());
                     }
                     CommandFragmentChanges::Reschedule { new_actors, .. } => {
                         let info = self
@@ -178,30 +155,19 @@ impl InflightActorInfo {
                             .expect("should exist");
                         let actors = &mut info.actors;
                         for (actor_id, node_id) in new_actors {
-                            assert!(to_add
-                                .insert(actor_id, (node_id, info.is_injectable))
-                                .is_none());
+                            assert!(to_add.insert(actor_id, node_id).is_none());
                             assert!(actors.insert(actor_id, node_id).is_none());
                         }
                     }
                     CommandFragmentChanges::RemoveFragment => {}
                 }
             }
-            for (actor_id, (node_id, is_injectable)) in to_add {
+            for (actor_id, node_id) in to_add {
                 assert!(self.node_map.contains_key(&node_id));
                 assert!(
                     self.actor_map.entry(node_id).or_default().insert(actor_id),
                     "duplicate actor in command changes"
                 );
-                if is_injectable {
-                    assert!(
-                        self.actor_map_to_send
-                            .entry(node_id)
-                            .or_default()
-                            .insert(actor_id),
-                        "duplicate actor in command changes"
-                    );
-                }
                 assert!(
                     self.actor_location_map.insert(actor_id, node_id).is_none(),
                     "duplicate actor in command changes"
@@ -232,7 +198,7 @@ impl InflightActorInfo {
             let mut all_to_remove = HashSet::new();
             for (fragment_id, changes) in fragment_changes.fragment_changes {
                 match changes {
-                    CommandFragmentChanges::NewFragment { .. } => {}
+                    CommandFragmentChanges::NewFragment(_) => {}
                     CommandFragmentChanges::Reschedule { to_remove, .. } => {
                         let info = self
                             .fragment_infos
@@ -261,13 +227,8 @@ impl InflightActorInfo {
                     .expect("actor not found");
                 let actor_ids = self.actor_map.get_mut(&node_id).expect("node not found");
                 assert!(actor_ids.remove(&actor_id), "actor not found");
-                self.actor_map_to_send
-                    .get_mut(&node_id)
-                    .map(|actor_ids| actor_ids.remove(&actor_id));
             }
             self.actor_map.retain(|_, actor_ids| !actor_ids.is_empty());
-            self.actor_map_to_send
-                .retain(|_, actor_ids| !actor_ids.is_empty());
         }
         if let Command::DropSubscription {
             subscription_id,
@@ -291,27 +252,49 @@ impl InflightActorInfo {
     }
 
     /// Returns actor list to collect in the target worker node.
-    pub fn actor_ids_to_collect(&self, node_id: &WorkerId) -> impl Iterator<Item = ActorId> {
-        self.actor_map
-            .get(node_id)
-            .cloned()
-            .unwrap_or_default()
-            .into_iter()
+    pub fn actor_ids_to_collect(
+        fragment_infos: &HashMap<FragmentId, InflightFragmentInfo>,
+        node_id: WorkerId,
+    ) -> impl Iterator<Item = ActorId> + '_ {
+        fragment_infos.values().flat_map(move |info| {
+            info.actors
+                .iter()
+                .filter_map(move |(actor_id, actor_node_id)| {
+                    if *actor_node_id == node_id {
+                        Some(*actor_id)
+                    } else {
+                        None
+                    }
+                })
+        })
     }
 
     /// Returns actor list to send in the target worker node.
-    pub fn actor_ids_to_send(&self, node_id: &WorkerId) -> impl Iterator<Item = ActorId> {
-        self.actor_map_to_send
-            .get(node_id)
-            .cloned()
-            .unwrap_or_default()
-            .into_iter()
+    pub fn actor_ids_to_send(
+        fragment_infos: &HashMap<FragmentId, InflightFragmentInfo>,
+        node_id: WorkerId,
+    ) -> impl Iterator<Item = ActorId> + '_ {
+        fragment_infos
+            .values()
+            .filter(|info| info.is_injectable)
+            .flat_map(move |info| {
+                info.actors
+                    .iter()
+                    .filter_map(move |(actor_id, actor_node_id)| {
+                        if *actor_node_id == node_id {
+                            Some(*actor_id)
+                        } else {
+                            None
+                        }
+                    })
+            })
     }
 
-    pub fn existing_table_ids(&self) -> HashSet<TableId> {
-        self.fragment_infos
+    pub fn existing_table_ids(
+        fragment_infos: &HashMap<FragmentId, InflightFragmentInfo>,
+    ) -> impl Iterator<Item = TableId> + '_ {
+        fragment_infos
             .values()
             .flat_map(|info| info.state_table_ids.iter().cloned())
-            .collect()
     }
 }
diff --git a/src/meta/src/barrier/mod.rs b/src/meta/src/barrier/mod.rs
index bb6737735dd44..e2a1b24794c6b 100644
--- a/src/meta/src/barrier/mod.rs
+++ b/src/meta/src/barrier/mod.rs
@@ -23,37 +23,39 @@ use std::time::Duration;
 use anyhow::Context;
 use arc_swap::ArcSwap;
 use fail::fail_point;
+use futures::future::try_join_all;
 use itertools::Itertools;
 use prometheus::HistogramTimer;
-use risingwave_common::bail;
 use risingwave_common::catalog::TableId;
 use risingwave_common::system_param::reader::SystemParamsRead;
 use risingwave_common::system_param::PAUSE_ON_NEXT_BOOTSTRAP_KEY;
 use risingwave_common::util::epoch::{Epoch, INVALID_EPOCH};
+use risingwave_common::{bail, must_match};
 use risingwave_hummock_sdk::change_log::build_table_change_log_delta;
+use risingwave_hummock_sdk::table_stats::from_prost_table_stats_map;
 use risingwave_hummock_sdk::table_watermark::{
     merge_multiple_new_table_watermarks, TableWatermarks,
 };
-use risingwave_hummock_sdk::{ExtendedSstableInfo, HummockSstableObjectId};
+use risingwave_hummock_sdk::{HummockSstableObjectId, LocalSstableInfo};
 use risingwave_pb::catalog::table::TableType;
 use risingwave_pb::ddl_service::DdlProgress;
+use risingwave_pb::hummock::HummockVersionStats;
 use risingwave_pb::meta::subscribe_response::{Info, Operation};
-use risingwave_pb::meta::PausedReason;
+use risingwave_pb::meta::{PausedReason, PbRecoveryStatus};
 use risingwave_pb::stream_service::barrier_complete_response::CreateMviewProgress;
 use risingwave_pb::stream_service::BarrierCompleteResponse;
 use thiserror_ext::AsReport;
 use tokio::sync::oneshot::{Receiver, Sender};
-use tokio::sync::Mutex;
+use tokio::sync::{mpsc, oneshot};
 use tokio::task::JoinHandle;
 use tracing::{error, info, warn, Instrument};
 
 use self::command::CommandContext;
 use self::notifier::Notifier;
-use self::progress::TrackingCommand;
 use crate::barrier::info::InflightActorInfo;
 use crate::barrier::notifier::BarrierInfo;
-use crate::barrier::progress::CreateMviewProgressTracker;
-use crate::barrier::rpc::ControlStreamManager;
+use crate::barrier::progress::{CreateMviewProgressTracker, TrackingJob};
+use crate::barrier::rpc::{merge_node_rpc_errors, ControlStreamManager};
 use crate::barrier::state::BarrierManagerState;
 use crate::error::MetaErrorInner;
 use crate::hummock::{CommitEpochInfo, HummockManagerRef, NewTableFragmentInfo};
@@ -62,7 +64,6 @@ use crate::manager::{
     ActiveStreamingWorkerChange, ActiveStreamingWorkerNodes, LocalNotification, MetaSrvEnv,
     MetadataManager, SystemParamsManagerImpl, WorkerId,
 };
-use crate::model::{ActorId, TableFragments};
 use crate::rpc::metrics::MetaMetrics;
 use crate::stream::{ScaleControllerRef, SourceManagerRef};
 use crate::{MetaError, MetaResult};
@@ -77,7 +78,9 @@ mod schedule;
 mod state;
 mod trace;
 
-pub use self::command::{BarrierKind, Command, ReplaceTablePlan, Reschedule};
+pub use self::command::{
+    BarrierKind, Command, CreateStreamingJobCommandInfo, ReplaceTablePlan, Reschedule,
+};
 pub use self::rpc::StreamRpcManager;
 pub use self::schedule::BarrierScheduler;
 pub use self::trace::TracedEpoch;
@@ -87,12 +90,6 @@ pub(crate) struct TableMap<T> {
     inner: HashMap<TableId, T>,
 }
 
-impl<T> TableMap<T> {
-    pub fn remove(&mut self, table_id: &TableId) -> Option<T> {
-        self.inner.remove(table_id)
-    }
-}
-
 impl<T> From<HashMap<TableId, T>> for TableMap<T> {
     fn from(inner: HashMap<TableId, T>) -> Self {
         Self { inner }
@@ -105,12 +102,6 @@ impl<T> From<TableMap<T>> for HashMap<TableId, T> {
     }
 }
 
-pub(crate) type TableActorMap = TableMap<HashSet<ActorId>>;
-pub(crate) type TableUpstreamMvCountMap = TableMap<HashMap<TableId, usize>>;
-pub(crate) type TableDefinitionMap = TableMap<String>;
-pub(crate) type TableNotifierMap = TableMap<Notifier>;
-pub(crate) type TableFragmentMap = TableMap<TableFragments>;
-
 /// The reason why the cluster is recovering.
 enum RecoveryReason {
     /// After bootstrap.
@@ -141,11 +132,28 @@ struct Scheduled {
     checkpoint: bool,
 }
 
+impl From<&BarrierManagerStatus> for PbRecoveryStatus {
+    fn from(status: &BarrierManagerStatus) -> Self {
+        match status {
+            BarrierManagerStatus::Starting => Self::StatusStarting,
+            BarrierManagerStatus::Recovering(reason) => match reason {
+                RecoveryReason::Bootstrap => Self::StatusStarting,
+                RecoveryReason::Failover(_) | RecoveryReason::Adhoc => Self::StatusRecovering,
+            },
+            BarrierManagerStatus::Running => Self::StatusRunning,
+        }
+    }
+}
+
+pub enum BarrierManagerRequest {
+    GetDdlProgress(Sender<HashMap<u32, DdlProgress>>),
+}
+
 #[derive(Clone)]
 pub struct GlobalBarrierManagerContext {
     status: Arc<ArcSwap<BarrierManagerStatus>>,
 
-    tracker: Arc<Mutex<CreateMviewProgressTracker>>,
+    request_tx: mpsc::UnboundedSender<BarrierManagerRequest>,
 
     metadata_manager: MetadataManager,
 
@@ -191,6 +199,8 @@ pub struct GlobalBarrierManager {
 
     checkpoint_control: CheckpointControl,
 
+    request_rx: mpsc::UnboundedReceiver<BarrierManagerRequest>,
+
     /// The `prev_epoch` of pending non checkpoint barriers
     pending_non_checkpoint_barriers: Vec<u64>,
 
@@ -209,14 +219,23 @@ struct CheckpointControl {
     /// The join handle of the completing future is stored.
     completing_command: CompletingCommand,
 
+    hummock_version_stats: HummockVersionStats,
+
+    create_mview_tracker: CreateMviewProgressTracker,
+
     context: GlobalBarrierManagerContext,
 }
 
 impl CheckpointControl {
-    fn new(context: GlobalBarrierManagerContext) -> Self {
+    async fn new(
+        context: GlobalBarrierManagerContext,
+        create_mview_tracker: CreateMviewProgressTracker,
+    ) -> Self {
         Self {
             command_ctx_queue: Default::default(),
             completing_command: CompletingCommand::None,
+            hummock_version_stats: context.hummock_manager.get_version_stats().await,
+            create_mview_tracker,
             context,
         }
     }
@@ -274,12 +293,10 @@ impl CheckpointControl {
 
     /// Change the state of this `prev_epoch` to `Completed`. Return continuous nodes
     /// with `Completed` starting from first node [`Completed`..`InFlight`) and remove them.
-    fn barrier_collected(
-        &mut self,
-        worker_id: WorkerId,
-        prev_epoch: u64,
-        resp: BarrierCompleteResponse,
-    ) {
+    fn barrier_collected(&mut self, resp: BarrierCompleteResponse) {
+        let worker_id = resp.worker_id;
+        let prev_epoch = resp.epoch;
+        assert_eq!(resp.partial_graph_id, u32::MAX);
         if let Some(node) = self.command_ctx_queue.get_mut(&prev_epoch) {
             assert!(node.state.node_to_collect.remove(&worker_id));
             node.state.resps.push(resp);
@@ -337,6 +354,7 @@ impl CheckpointControl {
             CompletingCommand::Completing {
                 command_ctx,
                 join_handle,
+                ..
             } => {
                 info!(
                     prev_epoch = ?command_ctx.prev_epoch,
@@ -363,20 +381,30 @@ impl CheckpointControl {
                 && !state.is_inflight()
             {
                 let (_, node) = self.command_ctx_queue.pop_first().expect("non-empty");
-                let command_ctx = node.command_ctx.clone();
-                if let Err(e) = self.context.clone().complete_barrier(node).await {
+                let (prev_epoch, curr_epoch) = (
+                    node.command_ctx.prev_epoch.value().0,
+                    node.command_ctx.curr_epoch.value().0,
+                );
+                let finished_jobs = self
+                    .create_mview_tracker
+                    .apply_collected_command(&node, &self.hummock_version_stats);
+                if let Err(e) = self
+                    .context
+                    .clone()
+                    .complete_barrier(node, finished_jobs)
+                    .await
+                {
                     error!(
-                        prev_epoch = ?command_ctx.prev_epoch,
-                        curr_epoch = ?command_ctx.curr_epoch,
+                        prev_epoch,
+                        curr_epoch,
                         err = ?e.as_report(),
                         "failed to complete barrier during recovery"
                     );
                     break;
                 } else {
                     info!(
-                        prev_epoch = ?command_ctx.prev_epoch,
-                        curr_epoch = ?command_ctx.curr_epoch,
-                        "succeed to complete barrier during recovery"
+                        prev_epoch,
+                        curr_epoch, "succeed to complete barrier during recovery"
                     )
                 }
             }
@@ -387,11 +415,22 @@ impl CheckpointControl {
             }
             node.enqueue_time.observe_duration();
         }
+        self.create_mview_tracker.abort_all();
+    }
+
+    /// Return the earliest command waiting on the `worker_id`.
+    fn command_wait_collect_from_worker(&self, worker_id: WorkerId) -> Option<&CommandContext> {
+        for epoch_node in self.command_ctx_queue.values() {
+            if epoch_node.state.node_to_collect.contains(&worker_id) {
+                return Some(&epoch_node.command_ctx);
+            }
+        }
+        None
     }
 }
 
 /// The state and message of this barrier, a node for concurrent checkpoint.
-pub struct EpochNode {
+struct EpochNode {
     /// Timer for recording barrier latency, taken after `complete_barriers`.
     enqueue_time: HistogramTimer,
 
@@ -420,11 +459,12 @@ enum CompletingCommand {
     None,
     Completing {
         command_ctx: Arc<CommandContext>,
+        require_next_checkpoint: bool,
 
         // The join handle of a spawned task that completes the barrier.
         // The return value indicate whether there is some create streaming job command
         // that has finished but not checkpointed. If there is any, we will force checkpoint on the next barrier
-        join_handle: JoinHandle<MetaResult<BarrierCompleteOutput>>,
+        join_handle: JoinHandle<MetaResult<Option<HummockVersionStats>>>,
     },
     #[expect(dead_code)]
     Err(MetaError),
@@ -433,7 +473,7 @@ enum CompletingCommand {
 impl GlobalBarrierManager {
     /// Create a new [`crate::barrier::GlobalBarrierManager`].
     #[allow(clippy::too_many_arguments)]
-    pub fn new(
+    pub async fn new(
         scheduled_barriers: schedule::ScheduledBarriers,
         env: MetaSrvEnv,
         metadata_manager: MetadataManager,
@@ -455,23 +495,25 @@ impl GlobalBarrierManager {
 
         let active_streaming_nodes = ActiveStreamingWorkerNodes::uninitialized();
 
-        let tracker = CreateMviewProgressTracker::new();
+        let tracker = CreateMviewProgressTracker::default();
+
+        let (request_tx, request_rx) = mpsc::unbounded_channel();
 
         let context = GlobalBarrierManagerContext {
             status: Arc::new(ArcSwap::new(Arc::new(BarrierManagerStatus::Starting))),
+            request_tx,
             metadata_manager,
             hummock_manager,
             source_manager,
             scale_controller,
             sink_manager,
             metrics,
-            tracker: Arc::new(Mutex::new(tracker)),
             stream_rpc_manager,
             env: env.clone(),
         };
 
         let control_stream_manager = ControlStreamManager::new(context.clone());
-        let checkpoint_control = CheckpointControl::new(context.clone());
+        let checkpoint_control = CheckpointControl::new(context.clone(), tracker).await;
 
         Self {
             enable_recovery,
@@ -481,6 +523,7 @@ impl GlobalBarrierManager {
             env,
             state: initial_invalid_state,
             checkpoint_control,
+            request_rx,
             pending_non_checkpoint_barriers: Vec::new(),
             active_streaming_nodes,
             control_stream_manager,
@@ -574,11 +617,19 @@ impl GlobalBarrierManager {
             self.context
                 .set_status(BarrierManagerStatus::Recovering(RecoveryReason::Bootstrap));
             let span = tracing::info_span!("bootstrap_recovery", prev_epoch = prev_epoch.value().0);
+            crate::telemetry::report_event(
+                risingwave_pb::telemetry::TelemetryEventStage::Recovery,
+                "normal_recovery",
+                0,
+                None,
+                None,
+                None,
+            );
 
             let paused = self.take_pause_on_bootstrap().await.unwrap_or(false);
             let paused_reason = paused.then_some(PausedReason::Manual);
 
-            self.recovery(paused_reason).instrument(span).await;
+            self.recovery(paused_reason, None).instrument(span).await;
         }
 
         self.context.set_status(BarrierManagerStatus::Running);
@@ -601,6 +652,21 @@ impl GlobalBarrierManager {
                     break;
                 }
 
+                request = self.request_rx.recv() => {
+                    if let Some(request) = request {
+                        match request {
+                            BarrierManagerRequest::GetDdlProgress(result_tx) => {
+                                if result_tx.send(self.checkpoint_control.create_mview_tracker.gen_ddl_progress()).is_err() {
+                                    error!("failed to send get ddl progress");
+                                }
+                            }
+                        }
+                    } else {
+                        tracing::info!("end of request stream. meta node may be shutting down. Stop global barrier manager");
+                        return;
+                    }
+                }
+
                 changed_worker = self.active_streaming_nodes.changed() => {
                     #[cfg(debug_assertions)]
                     {
@@ -619,7 +685,7 @@ impl GlobalBarrierManager {
                                             id: node.id,
                                             r#type: node.r#type,
                                             host: node.host.clone(),
-                                            parallel_units: node.parallel_units.clone(),
+                                            parallelism: node.parallelism,
                                             property: node.property.clone(),
                                             resource: node.resource.clone(),
                                             ..Default::default()
@@ -673,14 +739,25 @@ impl GlobalBarrierManager {
                         _ => {}
                     }
                 }
-                resp_result = self.control_stream_manager.next_complete_barrier_response() => {
+                (worker_id, resp_result) = self.control_stream_manager.next_complete_barrier_response() => {
                     match resp_result {
-                        Ok((worker_id, prev_epoch, resp)) => {
-                            self.checkpoint_control.barrier_collected(worker_id, prev_epoch, resp);
+                        Ok(resp) => {
+                            self.checkpoint_control.barrier_collected(resp);
 
                         }
                         Err(e) => {
-                            self.failure_recovery(e).await;
+                            let failed_command = self.checkpoint_control.command_wait_collect_from_worker(worker_id);
+                            if failed_command.is_some()
+                                || self.state.inflight_actor_infos.actor_map.contains_key(&worker_id) {
+                                let errors = self.control_stream_manager.collect_errors(worker_id, e).await;
+                                let err = merge_node_rpc_errors("get error from control stream", errors);
+                                if let Some(failed_command) = failed_command {
+                                    self.context.report_collect_failure(failed_command, &err);
+                                }
+                                self.failure_recovery(err).await;
+                            } else {
+                                warn!(e = ?e.as_report(), worker_id, "no barrier to collect from worker, ignore err");
+                            }
                         }
                     }
                 }
@@ -753,10 +830,11 @@ impl GlobalBarrierManager {
 
         send_latency_timer.observe_duration();
 
-        let node_to_collect = match self
-            .control_stream_manager
-            .inject_barrier(command_ctx.clone())
-        {
+        let node_to_collect = match self.control_stream_manager.inject_barrier(
+            &command_ctx,
+            &command_ctx.info.fragment_infos,
+            Some(&self.state.inflight_actor_infos.fragment_infos),
+        ) {
             Ok(node_to_collect) => node_to_collect,
             Err(err) => {
                 for notifier in notifiers {
@@ -788,7 +866,6 @@ impl GlobalBarrierManager {
     }
 
     async fn failure_recovery(&mut self, err: MetaError) {
-        self.context.tracker.lock().await.abort_all(&err);
         self.checkpoint_control.clear_on_err(&err).await;
         self.pending_non_checkpoint_barriers.clear();
 
@@ -805,9 +882,18 @@ impl GlobalBarrierManager {
                 prev_epoch = prev_epoch.value().0
             );
 
+            crate::telemetry::report_event(
+                risingwave_pb::telemetry::TelemetryEventStage::Recovery,
+                "failure_recovery",
+                0,
+                None,
+                None,
+                None,
+            );
+
             // No need to clean dirty tables for barrier recovery,
             // The foreground stream job should cleanup their own tables.
-            self.recovery(None).instrument(span).await;
+            self.recovery(None, Some(err)).instrument(span).await;
             self.context.set_status(BarrierManagerStatus::Running);
         } else {
             panic!("failed to execute barrier: {}", err.as_report());
@@ -816,76 +902,84 @@ impl GlobalBarrierManager {
 
     async fn adhoc_recovery(&mut self) {
         let err = MetaErrorInner::AdhocRecovery.into();
-        self.context.tracker.lock().await.abort_all(&err);
         self.checkpoint_control.clear_on_err(&err).await;
 
-        if self.enable_recovery {
-            self.context
-                .set_status(BarrierManagerStatus::Recovering(RecoveryReason::Adhoc));
-            let latest_snapshot = self.context.hummock_manager.latest_snapshot();
-            let prev_epoch = TracedEpoch::new(latest_snapshot.committed_epoch.into()); // we can only recover from the committed epoch
-            let span = tracing::info_span!(
-                "adhoc_recovery",
-                error = %err.as_report(),
-                prev_epoch = prev_epoch.value().0
-            );
+        self.context
+            .set_status(BarrierManagerStatus::Recovering(RecoveryReason::Adhoc));
+        let latest_snapshot = self.context.hummock_manager.latest_snapshot();
+        let prev_epoch = TracedEpoch::new(latest_snapshot.committed_epoch.into()); // we can only recover from the committed epoch
+        let span = tracing::info_span!(
+            "adhoc_recovery",
+            error = %err.as_report(),
+            prev_epoch = prev_epoch.value().0
+        );
 
-            // No need to clean dirty tables for barrier recovery,
-            // The foreground stream job should cleanup their own tables.
-            self.recovery(None).instrument(span).await;
-            self.context.set_status(BarrierManagerStatus::Running);
-        } else {
-            panic!("failed to execute barrier: {}", err.as_report());
-        }
+        crate::telemetry::report_event(
+            risingwave_pb::telemetry::TelemetryEventStage::Recovery,
+            "adhoc_recovery",
+            0,
+            None,
+            None,
+            None,
+        );
+
+        // No need to clean dirty tables for barrier recovery,
+        // The foreground stream job should cleanup their own tables.
+        self.recovery(None, Some(err)).instrument(span).await;
+        self.context.set_status(BarrierManagerStatus::Running);
     }
 }
 
 impl GlobalBarrierManagerContext {
     /// Try to commit this node. If err, returns
-    async fn complete_barrier(self, node: EpochNode) -> MetaResult<BarrierCompleteOutput> {
+    async fn complete_barrier(
+        self,
+        node: EpochNode,
+        finished_jobs: Vec<TrackingJob>,
+    ) -> MetaResult<Option<HummockVersionStats>> {
         let EpochNode {
             command_ctx,
-            mut notifiers,
+            notifiers,
             enqueue_time,
             state,
             ..
         } = node;
         assert!(state.node_to_collect.is_empty());
-        let resps = state.resps;
         let wait_commit_timer = self.metrics.barrier_wait_commit_latency.start_timer();
-        let create_mview_progress = resps
-            .iter()
-            .flat_map(|resp| resp.create_mview_progress.iter().cloned())
-            .collect();
-        if let Err(e) = self.update_snapshot(&command_ctx, resps).await {
-            for notifier in notifiers {
-                notifier.notify_collection_failed(e.clone());
+
+        let result = self.update_snapshot(&command_ctx, state).await;
+
+        let version_stats = match result {
+            Ok(version_stats) => version_stats,
+            Err(e) => {
+                for notifier in notifiers {
+                    notifier.notify_collection_failed(e.clone());
+                }
+                return Err(e);
             }
-            return Err(e);
         };
-        notifiers.iter_mut().for_each(|notifier| {
+        notifiers.into_iter().for_each(|notifier| {
             notifier.notify_collected();
         });
-        let has_remaining = self
-            .update_tracking_jobs(notifiers, command_ctx.clone(), create_mview_progress)
-            .await?;
+        try_join_all(finished_jobs.into_iter().map(|finished_job| {
+            let metadata_manager = &self.metadata_manager;
+            async move { finished_job.pre_finish(metadata_manager).await }
+        }))
+        .await?;
         let duration_sec = enqueue_time.stop_and_record();
         self.report_complete_event(duration_sec, &command_ctx);
         wait_commit_timer.observe_duration();
         self.metrics
             .last_committed_barrier_time
             .set(command_ctx.curr_epoch.value().as_unix_secs() as i64);
-        Ok(BarrierCompleteOutput {
-            command_ctx,
-            require_next_checkpoint: has_remaining,
-        })
+        Ok(version_stats)
     }
 
     async fn update_snapshot(
         &self,
         command_ctx: &CommandContext,
-        resps: Vec<BarrierCompleteResponse>,
-    ) -> MetaResult<()> {
+        state: BarrierEpochState,
+    ) -> MetaResult<Option<HummockVersionStats>> {
         {
             {
                 let prev_epoch = command_ctx.prev_epoch.value().0;
@@ -899,7 +993,7 @@ impl GlobalBarrierManagerContext {
                 match &command_ctx.kind {
                     BarrierKind::Initial => {}
                     BarrierKind::Checkpoint(epochs) => {
-                        let commit_info = collect_commit_epoch_info(resps, command_ctx, epochs);
+                        let commit_info = collect_commit_epoch_info(state, command_ctx, epochs);
                         new_snapshot = self.hummock_manager.commit_epoch(commit_info).await?;
                     }
                     BarrierKind::Barrier => {
@@ -922,41 +1016,40 @@ impl GlobalBarrierManagerContext {
                             Info::HummockSnapshot(snapshot),
                         );
                 }
-                Ok(())
+                Ok(if command_ctx.kind.is_checkpoint() {
+                    Some(self.hummock_manager.get_version_stats().await)
+                } else {
+                    None
+                })
             }
         }
     }
+}
 
-    async fn update_tracking_jobs(
-        &self,
-        notifiers: Vec<Notifier>,
-        command_ctx: Arc<CommandContext>,
-        create_mview_progress: Vec<CreateMviewProgress>,
-    ) -> MetaResult<bool> {
+impl CreateMviewProgressTracker {
+    fn update_tracking_jobs<'a>(
+        &mut self,
+        info: Option<(&CreateStreamingJobCommandInfo, Option<&ReplaceTablePlan>)>,
+        create_mview_progress: impl Iterator<Item = &'a CreateMviewProgress>,
+        version_stats: &HummockVersionStats,
+    ) {
         {
             {
-                // Notify about collected.
-                let version_stats = self.hummock_manager.get_version_stats().await;
-                let mut tracker = self.tracker.lock().await;
-
                 // Save `finished_commands` for Create MVs.
                 let finished_commands = {
                     let mut commands = vec![];
                     // Add the command to tracker.
-                    if let Some(command) = tracker.add(
-                        TrackingCommand {
-                            context: command_ctx.clone(),
-                            notifiers,
-                        },
-                        &version_stats,
-                    ) {
+                    if let Some((create_job_info, replace_table)) = info
+                        && let Some(command) =
+                            self.add(create_job_info, replace_table, version_stats)
+                    {
                         // Those with no actors to track can be finished immediately.
                         commands.push(command);
                     }
                     // Update the progress of all commands.
                     for progress in create_mview_progress {
                         // Those with actors complete can be finished immediately.
-                        if let Some(command) = tracker.update(&progress, &version_stats) {
+                        if let Some(command) = self.update(progress, version_stats) {
                             tracing::trace!(?progress, "finish progress");
                             commands.push(command);
                         } else {
@@ -967,43 +1060,27 @@ impl GlobalBarrierManagerContext {
                 };
 
                 for command in finished_commands {
-                    tracker.stash_command_to_finish(command);
+                    self.stash_command_to_finish(command);
                 }
-
-                if let Some(table_id) = command_ctx.table_to_cancel() {
-                    // the cancelled command is possibly stashed in `finished_commands` and waiting
-                    // for checkpoint, we should also clear it.
-                    tracker.cancel_command(table_id);
-                }
-
-                let has_remaining_job = tracker
-                    .finish_jobs(command_ctx.kind.is_checkpoint())
-                    .await?;
-
-                Ok(has_remaining_job)
             }
         }
     }
+}
 
+impl GlobalBarrierManagerContext {
     fn report_complete_event(&self, duration_sec: f64, command_ctx: &CommandContext) {
-        {
-            {
-                {
-                    // Record barrier latency in event log.
-                    use risingwave_pb::meta::event_log;
-                    let event = event_log::EventBarrierComplete {
-                        prev_epoch: command_ctx.prev_epoch.value().0,
-                        cur_epoch: command_ctx.curr_epoch.value().0,
-                        duration_sec,
-                        command: command_ctx.command.to_string(),
-                        barrier_kind: command_ctx.kind.as_str_name().to_string(),
-                    };
-                    self.env
-                        .event_log_manager_ref()
-                        .add_event_logs(vec![event_log::Event::BarrierComplete(event)]);
-                }
-            }
-        }
+        // Record barrier latency in event log.
+        use risingwave_pb::meta::event_log;
+        let event = event_log::EventBarrierComplete {
+            prev_epoch: command_ctx.prev_epoch.value().0,
+            cur_epoch: command_ctx.curr_epoch.value().0,
+            duration_sec,
+            command: command_ctx.command.to_string(),
+            barrier_kind: command_ctx.kind.as_str_name().to_string(),
+        };
+        self.env
+            .event_log_manager_ref()
+            .add_event_logs(vec![event_log::Event::BarrierComplete(event)]);
     }
 }
 
@@ -1021,10 +1098,23 @@ impl CheckpointControl {
                 && !state.is_inflight()
             {
                 let (_, node) = self.command_ctx_queue.pop_first().expect("non-empty");
+                let finished_jobs = self
+                    .create_mview_tracker
+                    .apply_collected_command(&node, &self.hummock_version_stats);
                 let command_ctx = node.command_ctx.clone();
-                let join_handle = tokio::spawn(self.context.clone().complete_barrier(node));
+                let join_handle =
+                    tokio::spawn(self.context.clone().complete_barrier(node, finished_jobs));
+                let require_next_checkpoint =
+                    if self.create_mview_tracker.has_pending_finished_jobs() {
+                        self.command_ctx_queue
+                            .values()
+                            .all(|node| !node.command_ctx.kind.is_checkpoint())
+                    } else {
+                        false
+                    };
                 self.completing_command = CompletingCommand::Completing {
                     command_ctx,
+                    require_next_checkpoint,
                     join_handle,
                 };
             }
@@ -1038,12 +1128,27 @@ impl CheckpointControl {
             };
             // It's important to reset the completing_command after await no matter the result is err
             // or not, and otherwise the join handle will be polled again after ready.
-            if let Err(e) = &join_result {
-                self.completing_command = CompletingCommand::Err(e.clone());
+            let next_completing_command_status = if let Err(e) = &join_result {
+                CompletingCommand::Err(e.clone())
             } else {
-                self.completing_command = CompletingCommand::None;
-            }
-            join_result
+                CompletingCommand::None
+            };
+            let completed_command =
+                replace(&mut self.completing_command, next_completing_command_status);
+            join_result.map(move |version_stats| {
+                if let Some(new_version_stats) = version_stats {
+                    self.hummock_version_stats = new_version_stats;
+                }
+                must_match!(
+                    completed_command,
+                    CompletingCommand::Completing { command_ctx, require_next_checkpoint, .. } => {
+                        BarrierCompleteOutput {
+                            command_ctx,
+                            require_next_checkpoint,
+                        }
+                    }
+                )
+            })
         } else {
             pending().await
         }
@@ -1069,6 +1174,10 @@ impl GlobalBarrierManagerContext {
         }
     }
 
+    pub fn get_recovery_status(&self) -> PbRecoveryStatus {
+        (&**self.status.load()).into()
+    }
+
     /// Set barrier manager status.
     fn set_status(&self, new_status: BarrierManagerStatus) {
         self.status.store(Arc::new(new_status));
@@ -1101,8 +1210,14 @@ impl GlobalBarrierManagerContext {
         Ok(info)
     }
 
-    pub async fn get_ddl_progress(&self) -> Vec<DdlProgress> {
-        let mut ddl_progress = self.tracker.lock().await.gen_ddl_progress();
+    pub async fn get_ddl_progress(&self) -> MetaResult<Vec<DdlProgress>> {
+        let mut ddl_progress = {
+            let (tx, rx) = oneshot::channel();
+            self.request_tx
+                .send(BarrierManagerRequest::GetDdlProgress(tx))
+                .context("failed to send get ddl progress request")?;
+            rx.await.context("failed to receive get ddl progress")?
+        };
         // If not in tracker, means the first barrier not collected yet.
         // In that case just return progress 0.
         match &self.metadata_manager {
@@ -1138,51 +1253,51 @@ impl GlobalBarrierManagerContext {
             }
         }
 
-        ddl_progress.into_values().collect()
+        Ok(ddl_progress.into_values().collect())
     }
 }
 
 pub type BarrierManagerRef = GlobalBarrierManagerContext;
 
 fn collect_commit_epoch_info(
-    resps: Vec<BarrierCompleteResponse>,
+    state: BarrierEpochState,
     command_ctx: &CommandContext,
     epochs: &Vec<u64>,
 ) -> CommitEpochInfo {
+    let resps = state.resps;
     let mut sst_to_worker: HashMap<HummockSstableObjectId, WorkerId> = HashMap::new();
-    let mut synced_ssts: Vec<ExtendedSstableInfo> = vec![];
+    let mut synced_ssts: Vec<LocalSstableInfo> = vec![];
     let mut table_watermarks = Vec::with_capacity(resps.len());
     let mut old_value_ssts = Vec::with_capacity(resps.len());
     for resp in resps {
         let ssts_iter = resp.synced_sstables.into_iter().map(|grouped| {
             let sst_info = grouped.sst.expect("field not None");
-            sst_to_worker.insert(sst_info.get_object_id(), resp.worker_id);
-            ExtendedSstableInfo::new(
-                grouped.compaction_group_id,
-                sst_info,
-                grouped.table_stats_map,
+            sst_to_worker.insert(sst_info.object_id, resp.worker_id);
+            LocalSstableInfo::new(
+                sst_info.into(),
+                from_prost_table_stats_map(grouped.table_stats_map),
             )
         });
         synced_ssts.extend(ssts_iter);
         table_watermarks.push(resp.table_watermarks);
-        old_value_ssts.extend(resp.old_value_sstables);
+        old_value_ssts.extend(resp.old_value_sstables.into_iter().map(|s| s.into()));
     }
-    let new_table_fragment_info = if let Command::CreateStreamingJob {
-        table_fragments, ..
-    } = &command_ctx.command
-    {
-        Some(NewTableFragmentInfo {
-            table_id: table_fragments.table_id(),
-            mv_table_id: table_fragments.mv_table_id().map(TableId::new),
-            internal_table_ids: table_fragments
-                .internal_table_ids()
-                .into_iter()
-                .map(TableId::new)
-                .collect(),
-        })
-    } else {
-        None
-    };
+
+    let new_table_fragment_info =
+        if let Command::CreateStreamingJob { info, .. } = &command_ctx.command {
+            let table_fragments = &info.table_fragments;
+            Some(NewTableFragmentInfo {
+                table_id: table_fragments.table_id(),
+                mv_table_id: table_fragments.mv_table_id().map(TableId::new),
+                internal_table_ids: table_fragments
+                    .internal_table_ids()
+                    .into_iter()
+                    .map(TableId::new)
+                    .collect(),
+            })
+        } else {
+            None
+        };
 
     let table_new_change_log = build_table_change_log_delta(
         old_value_ssts.into_iter(),
@@ -1213,10 +1328,7 @@ fn collect_commit_epoch_info(
                     watermarks
                         .into_iter()
                         .map(|(table_id, watermarks)| {
-                            (
-                                TableId::new(table_id),
-                                TableWatermarks::from_protobuf(&watermarks),
-                            )
+                            (TableId::new(table_id), TableWatermarks::from(&watermarks))
                         })
                         .collect()
                 })
@@ -1225,7 +1337,11 @@ fn collect_commit_epoch_info(
         sst_to_worker,
         new_table_fragment_info,
         table_new_change_log,
-        BTreeMap::from_iter([(epoch, command_ctx.info.existing_table_ids())]),
+        BTreeMap::from_iter([(
+            epoch,
+            InflightActorInfo::existing_table_ids(&command_ctx.info.fragment_infos).collect(),
+        )]),
         epoch,
+        vec![],
     )
 }
diff --git a/src/meta/src/barrier/notifier.rs b/src/meta/src/barrier/notifier.rs
index d8df005600e4d..24f13050b030a 100644
--- a/src/meta/src/barrier/notifier.rs
+++ b/src/meta/src/barrier/notifier.rs
@@ -36,9 +36,6 @@ pub(crate) struct Notifier {
 
     /// Get notified when scheduled barrier is collected or failed.
     pub collected: Option<oneshot::Sender<MetaResult<()>>>,
-
-    /// Get notified when scheduled barrier is finished.
-    pub finished: Option<oneshot::Sender<MetaResult<()>>>,
 }
 
 impl Notifier {
@@ -50,8 +47,8 @@ impl Notifier {
     }
 
     /// Notify when we have collected a barrier from all actors.
-    pub fn notify_collected(&mut self) {
-        if let Some(tx) = self.collected.take() {
+    pub fn notify_collected(self) {
+        if let Some(tx) = self.collected {
             tx.send(Ok(())).ok();
         }
     }
@@ -63,31 +60,10 @@ impl Notifier {
         }
     }
 
-    /// Notify when we have finished a barrier from all actors. This function consumes `self`.
-    ///
-    /// Generally when a barrier is collected, it's also finished since it does not require further
-    /// report of finishing from actors.
-    /// However for creating MV, this is only called when all `BackfillExecutor` report it finished.
-    pub fn notify_finished(self) {
-        if let Some(tx) = self.finished {
-            tx.send(Ok(())).ok();
-        }
-    }
-
-    /// Notify when we failed to finish a barrier. This function consumes `self`.
-    pub fn notify_finish_failed(self, err: MetaError) {
-        if let Some(tx) = self.finished {
-            tx.send(Err(err)).ok();
-        }
-    }
-
     /// Notify when we failed to collect or finish a barrier. This function consumes `self`.
     pub fn notify_failed(self, err: MetaError) {
         if let Some(tx) = self.collected {
             tx.send(Err(err.clone())).ok();
         }
-        if let Some(tx) = self.finished {
-            tx.send(Err(err)).ok();
-        }
     }
 }
diff --git a/src/meta/src/barrier/progress.rs b/src/meta/src/barrier/progress.rs
index 746a263b06317..2f11ebadbf3a7 100644
--- a/src/meta/src/barrier/progress.rs
+++ b/src/meta/src/barrier/progress.rs
@@ -13,25 +13,23 @@
 // limitations under the License.
 
 use std::collections::hash_map::Entry;
-use std::collections::{HashMap, HashSet};
-use std::sync::Arc;
+use std::collections::HashMap;
+use std::mem::take;
 
 use risingwave_common::catalog::TableId;
 use risingwave_common::util::epoch::Epoch;
-use risingwave_pb::catalog::CreateType;
+use risingwave_meta_model_v2::ObjectId;
+use risingwave_pb::catalog::{CreateType, Table};
 use risingwave_pb::ddl_service::DdlProgress;
 use risingwave_pb::hummock::HummockVersionStats;
 use risingwave_pb::stream_service::barrier_complete_response::CreateMviewProgress;
 
-use super::command::CommandContext;
-use super::notifier::Notifier;
-use crate::barrier::{
-    Command, TableActorMap, TableDefinitionMap, TableFragmentMap, TableNotifierMap,
-    TableUpstreamMvCountMap,
+use crate::barrier::{Command, CreateStreamingJobCommandInfo, EpochNode, ReplaceTablePlan};
+use crate::manager::{
+    DdlType, MetadataManager, MetadataManagerV1, MetadataManagerV2, StreamingJob,
 };
-use crate::manager::{DdlType, MetadataManager};
 use crate::model::{ActorId, TableFragments};
-use crate::{MetaError, MetaResult};
+use crate::MetaResult;
 
 type ConsumedRows = u64;
 
@@ -92,6 +90,7 @@ impl Progress {
     fn update(&mut self, actor: ActorId, new_state: BackfillState, upstream_total_key_count: u64) {
         self.upstream_total_key_count = upstream_total_key_count;
         let total_actors = self.states.len();
+        tracing::debug!(?actor, states = ?self.states, "update progress for actor");
         match self.states.remove(&actor).unwrap() {
             BackfillState::Init => {}
             BackfillState::ConsumingUpstream(_, old_consumed_rows) => {
@@ -155,86 +154,72 @@ impl Progress {
 ///    On recovery, the barrier manager will recover and start managing the job.
 pub enum TrackingJob {
     New(TrackingCommand),
-    Recovered(RecoveredTrackingJob),
+    RecoveredV1(RecoveredTrackingJobV1),
+    RecoveredV2(RecoveredTrackingJobV2),
 }
 
 impl TrackingJob {
-    fn metadata_manager(&self) -> &MetadataManager {
-        match self {
-            TrackingJob::New(command) => command.context.metadata_manager(),
-            TrackingJob::Recovered(recovered) => &recovered.metadata_manager,
-        }
-    }
-
-    /// Returns whether the `TrackingJob` requires a checkpoint to complete.
-    pub(crate) fn is_checkpoint_required(&self) -> bool {
-        match self {
-            // Recovered tracking job is always a streaming job,
-            // It requires a checkpoint to complete.
-            TrackingJob::Recovered(_) => true,
+    pub(crate) async fn pre_finish(&self, metadata_manager: &MetadataManager) -> MetaResult<()> {
+        match &self {
             TrackingJob::New(command) => {
-                command.context.kind.is_initial() || command.context.kind.is_checkpoint()
-            }
-        }
-    }
-
-    pub(crate) async fn pre_finish(&self) -> MetaResult<()> {
-        let table_fragments = match &self {
-            TrackingJob::New(command) => match &command.context.command {
-                Command::CreateStreamingJob {
-                    table_fragments, ..
-                } => Some(table_fragments),
-                _ => None,
-            },
-            TrackingJob::Recovered(recovered) => Some(&recovered.fragments),
-        };
-        // Update the state of the table fragments from `Creating` to `Created`, so that the
-        // fragments can be scaled.
-        if let Some(table_fragments) = table_fragments {
-            match self.metadata_manager() {
-                MetadataManager::V1(mgr) => {
-                    mgr.fragment_manager
-                        .mark_table_fragments_created(table_fragments.table_id())
-                        .await?;
+                let CreateStreamingJobCommandInfo {
+                    table_fragments,
+                    streaming_job,
+                    internal_tables,
+                    ..
+                } = &command.info;
+                match metadata_manager {
+                    MetadataManager::V1(mgr) => {
+                        mgr.fragment_manager
+                            .mark_table_fragments_created(table_fragments.table_id())
+                            .await?;
+                        mgr.catalog_manager
+                            .finish_stream_job(streaming_job.clone(), internal_tables.clone())
+                            .await?;
+                        Ok(())
+                    }
+                    MetadataManager::V2(mgr) => {
+                        mgr.catalog_controller
+                            .finish_streaming_job(
+                                streaming_job.id() as i32,
+                                command.replace_table_info.clone(),
+                            )
+                            .await?;
+                        Ok(())
+                    }
                 }
-                MetadataManager::V2(_) => {}
-            }
-        }
-        Ok(())
-    }
-
-    pub(crate) fn notify_finished(self) {
-        match self {
-            TrackingJob::New(command) => {
-                command
-                    .notifiers
-                    .into_iter()
-                    .for_each(Notifier::notify_finished);
             }
-            TrackingJob::Recovered(recovered) => {
-                recovered.finished.notify_finished();
-            }
-        }
-    }
-
-    pub(crate) fn notify_finish_failed(self, err: MetaError) {
-        match self {
-            TrackingJob::New(command) => {
-                command
-                    .notifiers
-                    .into_iter()
-                    .for_each(|n| n.notify_finish_failed(err.clone()));
+            TrackingJob::RecoveredV1(recovered) => {
+                let manager = &recovered.metadata_manager;
+                manager
+                    .fragment_manager
+                    .mark_table_fragments_created(recovered.fragments.table_id())
+                    .await?;
+                manager
+                    .catalog_manager
+                    .finish_stream_job(
+                        recovered.streaming_job.clone(),
+                        recovered.internal_tables.clone(),
+                    )
+                    .await?;
+                Ok(())
             }
-            TrackingJob::Recovered(recovered) => {
-                recovered.finished.notify_finish_failed(err);
+            TrackingJob::RecoveredV2(recovered) => {
+                recovered
+                    .metadata_manager
+                    .catalog_controller
+                    .finish_streaming_job(recovered.id, None)
+                    .await?;
+                Ok(())
             }
         }
     }
 
-    pub(crate) fn table_to_create(&self) -> Option<TableId> {
+    pub(crate) fn table_to_create(&self) -> TableId {
         match self {
-            TrackingJob::New(command) => command.context.table_to_create(),
-            TrackingJob::Recovered(recovered) => Some(recovered.fragments.table_id()),
+            TrackingJob::New(command) => command.info.table_fragments.table_id(),
+            TrackingJob::RecoveredV1(recovered) => recovered.fragments.table_id(),
+            TrackingJob::RecoveredV2(recovered) => (recovered.id as u32).into(),
         }
     }
 }
@@ -245,42 +230,46 @@ impl std::fmt::Debug for TrackingJob {
             TrackingJob::New(command) => write!(
                 f,
                 "TrackingJob::New({:?})",
-                command.context.table_to_create()
+                command.info.table_fragments.table_id()
             ),
-            TrackingJob::Recovered(recovered) => {
+            TrackingJob::RecoveredV1(recovered) => {
                 write!(
                     f,
-                    "TrackingJob::Recovered({:?})",
+                    "TrackingJob::RecoveredV1({:?})",
                     recovered.fragments.table_id()
                 )
             }
+            TrackingJob::RecoveredV2(recovered) => {
+                write!(f, "TrackingJob::RecoveredV2({:?})", recovered.id)
+            }
         }
     }
 }
 
-pub struct RecoveredTrackingJob {
+pub struct RecoveredTrackingJobV1 {
     pub fragments: TableFragments,
-    pub finished: Notifier,
-    pub metadata_manager: MetadataManager,
+    pub streaming_job: StreamingJob,
+    pub internal_tables: Vec<Table>,
+    pub metadata_manager: MetadataManagerV1,
+}
+
+pub struct RecoveredTrackingJobV2 {
+    pub id: ObjectId,
+    pub metadata_manager: MetadataManagerV2,
 }
 
 /// The command tracking by the [`CreateMviewProgressTracker`].
 pub(super) struct TrackingCommand {
-    /// The context of the command.
-    pub context: Arc<CommandContext>,
-
-    /// Should be called when the command is finished.
-    pub notifiers: Vec<Notifier>,
+    pub info: CreateStreamingJobCommandInfo,
+    pub replace_table_info: Option<ReplaceTablePlan>,
 }
 
-/// Track the progress of all creating mviews. When creation is done, `notify_finished` will be
-/// called on registered notifiers.
-///
 /// Tracking is done as follows:
 /// 1. We identify a `StreamJob` by its `TableId` of its `Materialized` table.
 /// 2. For each stream job, there are several actors which run its tasks.
 /// 3. With `progress_map` we can use the ID of the `StreamJob` to view its progress.
 /// 4. With `actor_map` we can use an actor's `ActorId` to find the ID of the `StreamJob`.
+#[derive(Default)]
 pub(super) struct CreateMviewProgressTracker {
     /// Progress of the create-mview DDL indicated by the `TableId`.
     progress_map: HashMap<TableId, (Progress, TrackingJob)>,
@@ -300,49 +289,40 @@ impl CreateMviewProgressTracker {
     /// Other state are persisted by the `BackfillExecutor`, such as:
     /// 1. `CreateMviewProgress`.
     /// 2. `Backfill` position.
-    pub fn recover(
-        table_map: TableActorMap,
-        mut upstream_mv_counts: TableUpstreamMvCountMap,
-        mut definitions: TableDefinitionMap,
+    pub fn recover_v1(
         version_stats: HummockVersionStats,
-        mut finished_notifiers: TableNotifierMap,
-        mut table_fragment_map: TableFragmentMap,
-        metadata_manager: MetadataManager,
+        mviews: HashMap<
+            TableId,
+            (
+                TableFragments,
+                Table,      // mview table
+                Vec<Table>, // internal tables
+            ),
+        >,
+        metadata_manager: MetadataManagerV1,
     ) -> Self {
         let mut actor_map = HashMap::new();
         let mut progress_map = HashMap::new();
-        let table_map: HashMap<_, HashSet<ActorId>> = table_map.into();
-        for (creating_table_id, actors) in table_map {
-            // 1. Recover `BackfillState` in the tracker.
+        for (creating_table_id, (table_fragments, mview, internal_tables)) in mviews {
+            let actors = table_fragments.backfill_actor_ids();
             let mut states = HashMap::new();
+            tracing::debug!(?actors, ?creating_table_id, "recover progress for actors");
             for actor in actors {
                 actor_map.insert(actor, creating_table_id);
                 states.insert(actor, BackfillState::ConsumingUpstream(Epoch(0), 0));
             }
-            let upstream_mv_count = upstream_mv_counts.remove(&creating_table_id).unwrap();
-            let upstream_total_key_count = upstream_mv_count
-                .iter()
-                .map(|(upstream_mv, count)| {
-                    *count as u64
-                        * version_stats
-                            .table_stats
-                            .get(&upstream_mv.table_id)
-                            .map_or(0, |stat| stat.total_key_count as u64)
-                })
-                .sum();
-            let definition = definitions.remove(&creating_table_id).unwrap();
-            let progress = Progress {
+
+            let progress = Self::recover_progress(
                 states,
-                done_count: 0, // Fill only after first barrier pass
-                upstream_mv_count,
-                upstream_total_key_count,
-                consumed_rows: 0, // Fill only after first barrier pass
-                definition,
-            };
-            let tracking_job = TrackingJob::Recovered(RecoveredTrackingJob {
-                fragments: table_fragment_map.remove(&creating_table_id).unwrap(),
-                finished: finished_notifiers.remove(&creating_table_id).unwrap(),
+                table_fragments.dependent_table_ids(),
+                mview.definition.clone(),
+                &version_stats,
+            );
+            let tracking_job = TrackingJob::RecoveredV1(RecoveredTrackingJobV1 {
+                fragments: table_fragments,
                 metadata_manager: metadata_manager.clone(),
+                internal_tables,
+                streaming_job: StreamingJob::MaterializedView(mview),
             });
             progress_map.insert(creating_table_id, (progress, tracking_job));
         }
@@ -353,14 +333,66 @@ impl CreateMviewProgressTracker {
         }
     }
 
-    pub fn new() -> Self {
+    pub fn recover_v2(
+        mview_map: HashMap<TableId, (String, TableFragments)>,
+        version_stats: HummockVersionStats,
+        metadata_manager: MetadataManagerV2,
+    ) -> Self {
+        let mut actor_map = HashMap::new();
+        let mut progress_map = HashMap::new();
+        for (creating_table_id, (definition, table_fragments)) in mview_map {
+            let mut states = HashMap::new();
+            let actors = table_fragments.backfill_actor_ids();
+            for actor in actors {
+                actor_map.insert(actor, creating_table_id);
+                states.insert(actor, BackfillState::ConsumingUpstream(Epoch(0), 0));
+            }
+
+            let progress = Self::recover_progress(
+                states,
+                table_fragments.dependent_table_ids(),
+                definition,
+                &version_stats,
+            );
+            let tracking_job = TrackingJob::RecoveredV2(RecoveredTrackingJobV2 {
+                id: creating_table_id.table_id as i32,
+                metadata_manager: metadata_manager.clone(),
+            });
+            progress_map.insert(creating_table_id, (progress, tracking_job));
+        }
         Self {
-            progress_map: Default::default(),
-            actor_map: Default::default(),
+            progress_map,
+            actor_map,
             finished_jobs: Vec::new(),
         }
     }
 
+    fn recover_progress(
+        states: HashMap<ActorId, BackfillState>,
+        upstream_mv_count: HashMap<TableId, usize>,
+        definition: String,
+        version_stats: &HummockVersionStats,
+    ) -> Progress {
+        let upstream_total_key_count = upstream_mv_count
+            .iter()
+            .map(|(upstream_mv, count)| {
+                *count as u64
+                    * version_stats
+                        .table_stats
+                        .get(&upstream_mv.table_id)
+                        .map_or(0, |stat| stat.total_key_count as u64)
+            })
+            .sum();
+        Progress {
+            states,
+            done_count: 0, // Fill only after first barrier pass
+            upstream_mv_count,
+            upstream_total_key_count,
+            consumed_rows: 0, // Fill only after first barrier pass
+            definition,
+        }
+    }
+
     pub fn gen_ddl_progress(&self) -> HashMap<u32, DdlProgress> {
         self.progress_map
             .iter()
@@ -376,6 +408,45 @@ impl CreateMviewProgressTracker {
             .collect()
     }
 
+    /// Apply a collected epoch node command to the tracker
+    /// Return the finished jobs when the barrier kind is `Checkpoint`
+    pub(super) fn apply_collected_command(
+        &mut self,
+        epoch_node: &EpochNode,
+        version_stats: &HummockVersionStats,
+    ) -> Vec<TrackingJob> {
+        let command_ctx = &epoch_node.command_ctx;
+        let new_tracking_job_info = if let Command::CreateStreamingJob {
+            info,
+            replace_table,
+        } = &command_ctx.command
+        {
+            Some((info, replace_table.as_ref()))
+        } else {
+            None
+        };
+        assert!(epoch_node.state.node_to_collect.is_empty());
+        self.update_tracking_jobs(
+            new_tracking_job_info,
+            epoch_node
+                .state
+                .resps
+                .iter()
+                .flat_map(|resp| resp.create_mview_progress.iter()),
+            version_stats,
+        );
+        if let Some(table_id) = command_ctx.command.table_to_cancel() {
+            // the cancelled command is possibly stashed in `finished_commands` and waiting
+            // for checkpoint, we should also clear it.
+            self.cancel_command(table_id);
+        }
+        if command_ctx.kind.is_checkpoint() {
+            self.take_finished_jobs()
+        } else {
+            vec![]
+        }
+    }
+
     /// Stash a command to finish later.
     pub(super) fn stash_command_to_finish(&mut self, finished_job: TrackingJob) {
         self.finished_jobs.push(finished_job);
@@ -386,35 +457,26 @@ impl CreateMviewProgressTracker {
     /// If not checkpoint, jobs which do not require checkpoint can be finished.
     ///
     /// Returns whether there are still remaining stashed jobs to finish.
-    pub(super) async fn finish_jobs(&mut self, checkpoint: bool) -> MetaResult<bool> {
+    pub(super) fn take_finished_jobs(&mut self) -> Vec<TrackingJob> {
         tracing::trace!(finished_jobs=?self.finished_jobs, progress_map=?self.progress_map, "finishing jobs");
-        for job in self
-            .finished_jobs
-            .extract_if(|job| checkpoint || !job.is_checkpoint_required())
-        {
-            // The command is ready to finish. We can now call `pre_finish`.
-            job.pre_finish().await?;
-            job.notify_finished();
-        }
-        Ok(!self.finished_jobs.is_empty())
+        take(&mut self.finished_jobs)
+    }
+
+    pub(super) fn has_pending_finished_jobs(&self) -> bool {
+        !self.finished_jobs.is_empty()
     }
 
     pub(super) fn cancel_command(&mut self, id: TableId) {
         let _ = self.progress_map.remove(&id);
-        self.finished_jobs
-            .retain(|x| x.table_to_create() != Some(id));
+        self.finished_jobs.retain(|x| x.table_to_create() != id);
         self.actor_map.retain(|_, table_id| *table_id != id);
     }
 
     /// Notify all tracked commands that error encountered and clear them.
-    pub fn abort_all(&mut self, err: &MetaError) {
+    pub fn abort_all(&mut self) {
         self.actor_map.clear();
-        self.finished_jobs.drain(..).for_each(|job| {
-            job.notify_finish_failed(err.clone());
-        });
-        self.progress_map
-            .drain()
-            .for_each(|(_, (_, job))| job.notify_finish_failed(err.clone()));
+        self.finished_jobs.clear();
+        self.progress_map.clear();
     }
 
     /// Add a new create-mview DDL command to track.
@@ -422,32 +484,38 @@ impl CreateMviewProgressTracker {
     /// If the actors to track is empty, return the given command as it can be finished immediately.
     pub fn add(
         &mut self,
-        command: TrackingCommand,
+        info: &CreateStreamingJobCommandInfo,
+        replace_table: Option<&ReplaceTablePlan>,
         version_stats: &HummockVersionStats,
     ) -> Option<TrackingJob> {
-        let actors = command.context.actors_to_track();
-        if actors.is_empty() {
-            // The command can be finished immediately.
-            return Some(TrackingJob::New(command));
-        }
+        let (info, actors, replace_table_info) = {
+            let CreateStreamingJobCommandInfo {
+                table_fragments, ..
+            } = info;
+            let actors = table_fragments.tracking_progress_actor_ids();
+            if actors.is_empty() {
+                // The command can be finished immediately.
+                return Some(TrackingJob::New(TrackingCommand {
+                    info: info.clone(),
+                    replace_table_info: replace_table.cloned(),
+                }));
+            }
+            (info.clone(), actors, replace_table.cloned())
+        };
 
-        let (
-            creating_mv_id,
-            upstream_mv_count,
-            upstream_total_key_count,
-            definition,
-            ddl_type,
-            create_type,
-        ) = if let Command::CreateStreamingJob {
+        let CreateStreamingJobCommandInfo {
             table_fragments,
-            dispatchers,
             upstream_root_actors,
+            dispatchers,
             definition,
             ddl_type,
             create_type,
             ..
-        } = &command.context.command
-        {
+        } = &info;
+
+        let creating_mv_id = table_fragments.table_id();
+
+        let (upstream_mv_count, upstream_total_key_count, ddl_type, create_type) = {
             // Keep track of how many times each upstream MV appears.
             let mut upstream_mv_count = HashMap::new();
             for (table_id, actors) in upstream_root_actors {
@@ -471,15 +539,11 @@ impl CreateMviewProgressTracker {
                 })
                 .sum();
             (
-                table_fragments.table_id(),
                 upstream_mv_count,
                 upstream_total_key_count,
-                definition.to_string(),
                 ddl_type,
                 create_type,
             )
-        } else {
-            unreachable!("Must be CreateStreamingJob.");
         };
 
         for &actor in &actors {
@@ -490,7 +554,7 @@ impl CreateMviewProgressTracker {
             actors,
             upstream_mv_count,
             upstream_total_key_count,
-            definition,
+            definition.clone(),
         );
         if *ddl_type == DdlType::Sink && *create_type == CreateType::Background {
             // We return the original tracking job immediately.
@@ -498,11 +562,21 @@ impl CreateMviewProgressTracker {
             // We don't need to wait for sink to finish backfill.
             // This still contains the notifiers, so we can tell listeners
             // that the sink job has been created.
-            Some(TrackingJob::New(command))
+            Some(TrackingJob::New(TrackingCommand {
+                info,
+                replace_table_info,
+            }))
         } else {
-            let old = self
-                .progress_map
-                .insert(creating_mv_id, (progress, TrackingJob::New(command)));
+            let old = self.progress_map.insert(
+                creating_mv_id,
+                (
+                    progress,
+                    TrackingJob::New(TrackingCommand {
+                        info,
+                        replace_table_info,
+                    }),
+                ),
+            );
             assert!(old.is_none());
             None
         }
@@ -554,6 +628,7 @@ impl CreateMviewProgressTracker {
                     })
                     .sum();
 
+                tracing::debug!(?table_id, "updating progress for table");
                 progress.update(actor, new_state, upstream_total_key_count);
 
                 if progress.is_done() {
diff --git a/src/meta/src/barrier/recovery.rs b/src/meta/src/barrier/recovery.rs
index 3bb51b3b5ef73..6fbd89d149150 100644
--- a/src/meta/src/barrier/recovery.rs
+++ b/src/meta/src/barrier/recovery.rs
@@ -14,12 +14,13 @@
 
 use std::collections::{BTreeSet, HashMap, HashSet};
 use std::sync::Arc;
-use std::time::{Duration, Instant};
+use std::time::Duration;
 
 use anyhow::{anyhow, Context};
 use itertools::Itertools;
 use risingwave_common::catalog::TableId;
 use risingwave_common::config::DefaultParallelism;
+use risingwave_common::hash::WorkerSlotId;
 use risingwave_meta_model_v2::StreamingParallelism;
 use risingwave_pb::common::ActorInfo;
 use risingwave_pb::meta::subscribe_response::{Info, Operation};
@@ -28,14 +29,13 @@ use risingwave_pb::meta::{PausedReason, Recovery};
 use risingwave_pb::stream_plan::barrier_mutation::Mutation;
 use risingwave_pb::stream_plan::AddMutation;
 use thiserror_ext::AsReport;
-use tokio::sync::oneshot;
+use tokio::time::Instant;
 use tokio_retry::strategy::{jitter, ExponentialBackoff};
 use tracing::{debug, warn, Instrument};
 
-use super::TracedEpoch;
+use super::{CheckpointControl, TracedEpoch};
 use crate::barrier::command::CommandContext;
 use crate::barrier::info::InflightActorInfo;
-use crate::barrier::notifier::Notifier;
 use crate::barrier::progress::CreateMviewProgressTracker;
 use crate::barrier::rpc::ControlStreamManager;
 use crate::barrier::schedule::ScheduledBarriers;
@@ -45,9 +45,11 @@ use crate::controller::catalog::ReleaseContext;
 use crate::manager::{ActiveStreamingWorkerNodes, MetadataManager, WorkerId};
 use crate::model::{MetadataModel, MigrationPlan, TableFragments, TableParallelism};
 use crate::stream::{build_actor_connector_splits, RescheduleOptions, TableResizePolicy};
-use crate::{model, MetaResult};
+use crate::{model, MetaError, MetaResult};
 
 impl GlobalBarrierManager {
+    // Migration timeout.
+    const RECOVERY_FORCE_MIGRATION_TIMEOUT: Duration = Duration::from_secs(300);
     // Retry base interval in milliseconds.
     const RECOVERY_RETRY_BASE_INTERVAL: u64 = 20;
     // Retry max interval.
@@ -120,23 +122,18 @@ impl GlobalBarrierManagerContext {
         Ok(())
     }
 
-    async fn recover_background_mv_progress(&self) -> MetaResult<()> {
+    async fn recover_background_mv_progress(&self) -> MetaResult<CreateMviewProgressTracker> {
         match &self.metadata_manager {
             MetadataManager::V1(_) => self.recover_background_mv_progress_v1().await,
             MetadataManager::V2(_) => self.recover_background_mv_progress_v2().await,
         }
     }
 
-    async fn recover_background_mv_progress_v1(&self) -> MetaResult<()> {
+    async fn recover_background_mv_progress_v1(&self) -> MetaResult<CreateMviewProgressTracker> {
         let mgr = self.metadata_manager.as_v1_ref();
         let mviews = mgr.catalog_manager.list_creating_background_mvs().await;
 
-        let mut mview_definitions = HashMap::new();
-        let mut table_map = HashMap::new();
-        let mut table_fragment_map = HashMap::new();
-        let mut upstream_mv_counts = HashMap::new();
-        let mut senders = HashMap::new();
-        let mut receivers = Vec::new();
+        let mut table_mview_map = HashMap::new();
         for mview in mviews {
             let table_id = TableId::new(mview.id);
             let fragments = mgr
@@ -152,144 +149,45 @@ impl GlobalBarrierManagerContext {
                     .await?;
                 tracing::debug!("notified frontend for stream job {}", table_id.table_id);
             } else {
-                table_map.insert(table_id, fragments.backfill_actor_ids());
-                mview_definitions.insert(table_id, mview.definition.clone());
-                upstream_mv_counts.insert(table_id, fragments.dependent_table_ids());
-                table_fragment_map.insert(table_id, fragments);
-                let (finished_tx, finished_rx) = oneshot::channel();
-                senders.insert(
-                    table_id,
-                    Notifier {
-                        finished: Some(finished_tx),
-                        ..Default::default()
-                    },
-                );
-                receivers.push((mview, internal_tables, finished_rx));
+                table_mview_map.insert(table_id, (fragments, mview, internal_tables));
             }
         }
 
         let version_stats = self.hummock_manager.get_version_stats().await;
         // If failed, enter recovery mode.
-        {
-            let mut tracker = self.tracker.lock().await;
-            *tracker = CreateMviewProgressTracker::recover(
-                table_map.into(),
-                upstream_mv_counts.into(),
-                mview_definitions.into(),
-                version_stats,
-                senders.into(),
-                table_fragment_map.into(),
-                self.metadata_manager.clone(),
-            );
-        }
-        for (table, internal_tables, finished) in receivers {
-            let catalog_manager = mgr.catalog_manager.clone();
-            tokio::spawn(async move {
-                let res: MetaResult<()> = try {
-                    tracing::debug!("recovering stream job {}", table.id);
-                    finished.await.ok().context("failed to finish command")??;
-
-                    tracing::debug!("finished stream job {}", table.id);
-                    // Once notified that job is finished we need to notify frontend.
-                    // and mark catalog as created and commit to meta.
-                    // both of these are done by catalog manager.
-                    catalog_manager
-                        .finish_create_materialized_view_procedure(internal_tables, table.clone())
-                        .await?;
-                    tracing::debug!("notified frontend for stream job {}", table.id);
-                };
-                if let Err(e) = res.as_ref() {
-                    tracing::error!(
-                        id = table.id,
-                        error = %e.as_report(),
-                        "stream job interrupted, will retry after recovery",
-                    );
-                    // NOTE(kwannoel): We should not cleanup stream jobs,
-                    // we don't know if it's just due to CN killed,
-                    // or the job has actually failed.
-                    // Users have to manually cancel the stream jobs,
-                    // if they want to clean it.
-                }
-            });
-        }
-        Ok(())
+        Ok(CreateMviewProgressTracker::recover_v1(
+            version_stats,
+            table_mview_map,
+            mgr.clone(),
+        ))
     }
 
-    async fn recover_background_mv_progress_v2(&self) -> MetaResult<()> {
+    async fn recover_background_mv_progress_v2(&self) -> MetaResult<CreateMviewProgressTracker> {
         let mgr = self.metadata_manager.as_v2_ref();
         let mviews = mgr
             .catalog_controller
             .list_background_creating_mviews()
             .await?;
 
-        let mut senders = HashMap::new();
-        let mut receivers = Vec::new();
-        let mut table_fragment_map = HashMap::new();
-        let mut mview_definitions = HashMap::new();
-        let mut table_map = HashMap::new();
-        let mut upstream_mv_counts = HashMap::new();
+        let mut mview_map = HashMap::new();
         for mview in &mviews {
-            let (finished_tx, finished_rx) = oneshot::channel();
             let table_id = TableId::new(mview.table_id as _);
-            senders.insert(
-                table_id,
-                Notifier {
-                    finished: Some(finished_tx),
-                    ..Default::default()
-                },
-            );
-
             let table_fragments = mgr
                 .catalog_controller
                 .get_job_fragments_by_id(mview.table_id)
                 .await?;
             let table_fragments = TableFragments::from_protobuf(table_fragments);
-            upstream_mv_counts.insert(table_id, table_fragments.dependent_table_ids());
-            table_map.insert(table_id, table_fragments.backfill_actor_ids());
-            table_fragment_map.insert(table_id, table_fragments);
-            mview_definitions.insert(table_id, mview.definition.clone());
-            receivers.push((mview.table_id, finished_rx));
+            mview_map.insert(table_id, (mview.definition.clone(), table_fragments));
         }
 
         let version_stats = self.hummock_manager.get_version_stats().await;
         // If failed, enter recovery mode.
-        {
-            let mut tracker = self.tracker.lock().await;
-            *tracker = CreateMviewProgressTracker::recover(
-                table_map.into(),
-                upstream_mv_counts.into(),
-                mview_definitions.into(),
-                version_stats,
-                senders.into(),
-                table_fragment_map.into(),
-                self.metadata_manager.clone(),
-            );
-        }
-        for (id, finished) in receivers {
-            let catalog_controller = mgr.catalog_controller.clone();
-            tokio::spawn(async move {
-                let res: MetaResult<()> = try {
-                    tracing::debug!("recovering stream job {}", id);
-                    finished.await.ok().context("failed to finish command")??;
-                    tracing::debug!(id, "finished stream job");
-                    catalog_controller.finish_streaming_job(id, None).await?;
-                };
-                if let Err(e) = &res {
-                    tracing::error!(
-                        id,
-                        error = %e.as_report(),
-                        "stream job interrupted, will retry after recovery",
-                    );
-                    // NOTE(kwannoel): We should not cleanup stream jobs,
-                    // we don't know if it's just due to CN killed,
-                    // or the job has actually failed.
-                    // Users have to manually cancel the stream jobs,
-                    // if they want to clean it.
-                }
-            });
-        }
 
-        Ok(())
+        Ok(CreateMviewProgressTracker::recover_v2(
+            mview_map,
+            version_stats,
+            mgr.clone(),
+        ))
     }
 
     /// Pre buffered drop and cancel command, return true if any.
@@ -329,7 +227,7 @@ impl GlobalBarrierManager {
     /// the cluster or `risectl` command. Used for debugging purpose.
     ///
     /// Returns the new state of the barrier manager after recovery.
-    pub async fn recovery(&mut self, paused_reason: Option<PausedReason>) {
+    pub async fn recovery(&mut self, paused_reason: Option<PausedReason>, err: Option<MetaError>) {
         let prev_epoch = TracedEpoch::new(
             self.context
                 .hummock_manager
@@ -337,9 +235,12 @@ impl GlobalBarrierManager {
                 .committed_epoch
                 .into(),
         );
+
         // Mark blocked and abort buffered schedules, they might be dirty already.
         self.scheduled_barriers
             .abort_and_mark_blocked("cluster is under recovering");
+        // Clear all control streams to release resources (connections to compute nodes) first.
+        self.control_stream_manager.clear();
 
         tracing::info!("recovery start!");
         let retry_strategy = Self::get_retry_strategy();
@@ -351,6 +252,13 @@ impl GlobalBarrierManager {
         let new_state = tokio_retry::Retry::spawn(retry_strategy, || {
             async {
                 let recovery_result: MetaResult<_> = try {
+                    if let Some(err) = &err {
+                        self.context
+                            .metadata_manager
+                            .notify_finish_failed(err)
+                            .await;
+                    }
+
                     self.context
                         .clean_dirty_streaming_jobs()
                         .await
@@ -358,7 +266,8 @@ impl GlobalBarrierManager {
 
                     // Mview progress needs to be recovered.
                     tracing::info!("recovering mview progress");
-                    self.context
+                    let tracker = self
+                        .context
                         .recover_background_mv_progress()
                         .await
                         .context("recover mview progress should not fail")?;
@@ -384,6 +293,7 @@ impl GlobalBarrierManager {
                     // Resolve actor info for recovery. If there's no actor to recover, most of the
                     // following steps will be no-op, while the compute nodes will still be reset.
                     // FIXME: Transactions should be used.
+                    // TODO(error-handling): attach context to the errors and log them together, instead of inspecting everywhere.
                     let mut info = if !self.env.opts.disable_automatic_parallelism_control
                         && background_streaming_jobs.is_empty()
                     {
@@ -427,7 +337,9 @@ impl GlobalBarrierManager {
                     let info = info;
 
                     self.context
-                        .purge_state_table_from_hummock(&info.existing_table_ids())
+                        .purge_state_table_from_hummock(
+                            &InflightActorInfo::existing_table_ids(&info.fragment_infos).collect(),
+                        )
                         .await
                         .context("purge state table from hummock")?;
 
@@ -460,6 +372,7 @@ impl GlobalBarrierManager {
                         added_actors: Default::default(),
                         actor_splits: build_actor_connector_splits(&source_split_assignments),
                         pause: paused_reason.is_some(),
+                        subscriptions_to_add: Default::default(),
                     })));
 
                     // Use a different `curr_epoch` for each recovery attempt.
@@ -477,13 +390,17 @@ impl GlobalBarrierManager {
                         tracing::Span::current(), // recovery span
                     ));
 
-                    let mut node_to_collect =
-                        control_stream_manager.inject_barrier(command_ctx.clone())?;
+                    let mut node_to_collect = control_stream_manager.inject_barrier(
+                        &command_ctx,
+                        &info.fragment_infos,
+                        Some(&info.fragment_infos),
+                    )?;
                     while !node_to_collect.is_empty() {
-                        let (worker_id, prev_epoch, _) = control_stream_manager
+                        let (worker_id, result) = control_stream_manager
                             .next_complete_barrier_response()
-                            .await?;
-                        assert_eq!(prev_epoch, command_ctx.prev_epoch.value().0);
+                            .await;
+                        let resp = result?;
+                        assert_eq!(resp.epoch, command_ctx.prev_epoch.value().0);
                         assert!(node_to_collect.remove(&worker_id));
                     }
 
@@ -491,6 +408,7 @@ impl GlobalBarrierManager {
                         BarrierManagerState::new(new_epoch, info, command_ctx.next_paused_reason()),
                         active_streaming_nodes,
                         control_stream_manager,
+                        tracker,
                     )
                 };
                 if recovery_result.is_err() {
@@ -506,12 +424,18 @@ impl GlobalBarrierManager {
         recovery_timer.observe_duration();
         self.scheduled_barriers.mark_ready();
 
+        let create_mview_tracker: CreateMviewProgressTracker;
+
         (
             self.state,
             self.active_streaming_nodes,
             self.control_stream_manager,
+            create_mview_tracker,
         ) = new_state;
 
+        self.checkpoint_control =
+            CheckpointControl::new(self.context.clone(), create_mview_tracker).await;
+
         tracing::info!(
             epoch = self.state.in_flight_prev_epoch().value().0,
             paused = ?self.state.paused_reason(),
@@ -542,88 +466,133 @@ impl GlobalBarrierManagerContext {
     ) -> MetaResult<InflightActorInfo> {
         let mgr = self.metadata_manager.as_v2_ref();
 
-        let all_inuse_parallel_units: HashSet<_> = mgr
+        // all worker slots used by actors
+        let all_inuse_worker_slots: HashSet<_> = mgr
             .catalog_controller
-            .all_inuse_parallel_units()
+            .all_inuse_worker_slots()
             .await?
             .into_iter()
             .collect();
 
-        let active_parallel_units: HashSet<_> = active_nodes
+        let active_worker_slots: HashSet<_> = active_nodes
             .current()
             .values()
-            .flat_map(|node| node.parallel_units.iter().map(|pu| pu.id as i32))
+            .flat_map(|node| {
+                (0..node.parallelism).map(|idx| WorkerSlotId::new(node.id, idx as usize))
+            })
             .collect();
 
-        let expired_parallel_units: BTreeSet<_> = all_inuse_parallel_units
-            .difference(&active_parallel_units)
+        let expired_worker_slots: BTreeSet<_> = all_inuse_worker_slots
+            .difference(&active_worker_slots)
             .cloned()
             .collect();
-        if expired_parallel_units.is_empty() {
-            debug!("no expired parallel units, skipping.");
+
+        if expired_worker_slots.is_empty() {
+            debug!("no expired worker slots, skipping.");
             return self.resolve_actor_info(active_nodes).await;
         }
 
         debug!("start migrate actors.");
-        let mut to_migrate_parallel_units = expired_parallel_units.into_iter().rev().collect_vec();
-        debug!(
-            "got to migrate parallel units {:#?}",
-            to_migrate_parallel_units
-        );
-        let mut inuse_parallel_units: HashSet<_> = all_inuse_parallel_units
-            .intersection(&active_parallel_units)
+        let mut to_migrate_worker_slots = expired_worker_slots.into_iter().rev().collect_vec();
+        debug!("got to migrate worker slots {:#?}", to_migrate_worker_slots);
+
+        let mut inuse_worker_slots: HashSet<_> = all_inuse_worker_slots
+            .intersection(&active_worker_slots)
             .cloned()
             .collect();
 
         let start = Instant::now();
         let mut plan = HashMap::new();
-        'discovery: while !to_migrate_parallel_units.is_empty() {
-            let new_parallel_units = active_nodes
+        'discovery: while !to_migrate_worker_slots.is_empty() {
+            let mut new_worker_slots = active_nodes
                 .current()
                 .values()
-                .flat_map(|node| {
-                    node.parallel_units
-                        .iter()
-                        .filter(|pu| !inuse_parallel_units.contains(&(pu.id as _)))
+                .flat_map(|worker| {
+                    (0..worker.parallelism).map(move |i| WorkerSlotId::new(worker.id, i as _))
                 })
-                .cloned()
                 .collect_vec();
-            if !new_parallel_units.is_empty() {
-                debug!("new parallel units found: {:#?}", new_parallel_units);
-                for target_parallel_unit in new_parallel_units {
-                    if let Some(from) = to_migrate_parallel_units.pop() {
+
+            new_worker_slots.retain(|worker_slot| !inuse_worker_slots.contains(worker_slot));
+            let to_migration_size = to_migrate_worker_slots.len();
+            let mut available_size = new_worker_slots.len();
+
+            if available_size < to_migration_size
+                && start.elapsed() > GlobalBarrierManager::RECOVERY_FORCE_MIGRATION_TIMEOUT
+            {
+                let mut factor = 2;
+
+                while available_size < to_migration_size {
+                    let mut extended_worker_slots = active_nodes
+                        .current()
+                        .values()
+                        .flat_map(|worker| {
+                            (0..worker.parallelism * factor)
+                                .map(move |i| WorkerSlotId::new(worker.id, i as _))
+                        })
+                        .collect_vec();
+
+                    extended_worker_slots
+                        .retain(|worker_slot| !inuse_worker_slots.contains(worker_slot));
+
+                    extended_worker_slots.sort_by(|a, b| {
+                        a.slot_idx()
+                            .cmp(&b.slot_idx())
+                            .then(a.worker_id().cmp(&b.worker_id()))
+                    });
+
+                    available_size = extended_worker_slots.len();
+                    new_worker_slots = extended_worker_slots;
+
+                    factor *= 2;
+                }
+
+                tracing::info!(
+                    "migration timed out, extending worker slots to {:?} by factor {}",
+                    new_worker_slots,
+                    factor,
+                );
+            }
+
+            if !new_worker_slots.is_empty() {
+                debug!("new worker slots found: {:#?}", new_worker_slots);
+                for target_worker_slot in new_worker_slots {
+                    if let Some(from) = to_migrate_worker_slots.pop() {
                         debug!(
-                            "plan to migrate from parallel unit {} to {}",
-                            from, target_parallel_unit.id
+                            "plan to migrate from worker slot {} to {}",
+                            from, target_worker_slot
                         );
-                        inuse_parallel_units.insert(target_parallel_unit.id as i32);
-                        plan.insert(from, target_parallel_unit);
+                        inuse_worker_slots.insert(target_worker_slot);
+                        plan.insert(from, target_worker_slot);
                     } else {
                         break 'discovery;
                     }
                 }
             }
 
-            if to_migrate_parallel_units.is_empty() {
+            if to_migrate_worker_slots.is_empty() {
                 break;
             }
 
             // wait to get newly joined CN
             let changed = active_nodes
-                .wait_changed(Duration::from_millis(5000), |active_nodes| {
-                    let current_nodes = active_nodes
-                        .current()
-                        .values()
-                        .map(|node| (node.id, &node.host, &node.parallel_units))
-                        .collect_vec();
-                    warn!(
-                        current_nodes = ?current_nodes,
-                        "waiting for new workers to join, elapsed: {}s",
-                        start.elapsed().as_secs()
-                    );
-                })
+                .wait_changed(
+                    Duration::from_millis(5000),
+                    GlobalBarrierManager::RECOVERY_FORCE_MIGRATION_TIMEOUT,
+                    |active_nodes| {
+                        let current_nodes = active_nodes
+                            .current()
+                            .values()
+                            .map(|node| (node.id, &node.host, node.parallelism))
+                            .collect_vec();
+                        warn!(
+                            current_nodes = ?current_nodes,
+                            "waiting for new workers to join, elapsed: {}s",
+                            start.elapsed().as_secs()
+                        );
+                    },
+                )
                 .await;
-            warn!(?changed, "get worker changed. Retry migrate");
+            warn!(?changed, "get worker changed or timed out. Retry migrate");
         }
 
         mgr.catalog_controller.migrate_actors(plan).await?;
@@ -686,8 +655,8 @@ impl GlobalBarrierManagerContext {
         let available_parallelism = active_nodes
             .current()
             .values()
-            .flat_map(|worker_node| worker_node.parallel_units.iter())
-            .count();
+            .map(|worker_node| worker_node.parallelism as usize)
+            .sum();
 
         let table_parallelisms: HashMap<_, _> = {
             let streaming_parallelisms = mgr
@@ -768,7 +737,7 @@ impl GlobalBarrierManagerContext {
         } else {
             let (reschedule_fragment, _) = self
                 .scale_controller
-                .prepare_reschedule_command(
+                .analyze_reschedule_plan(
                     plan,
                     RescheduleOptions {
                         resolve_no_shuffle_upstream: true,
@@ -854,11 +823,11 @@ impl GlobalBarrierManagerContext {
         let available_parallelism = info
             .node_map
             .values()
-            .flat_map(|worker_node| worker_node.parallel_units.iter())
-            .count();
+            .map(|worker_node| worker_node.parallelism as usize)
+            .sum();
 
         if available_parallelism == 0 {
-            return Err(anyhow!("no available parallel units for auto scaling").into());
+            return Err(anyhow!("no available worker slots for auto scaling").into());
         }
 
         let all_table_parallelisms: HashMap<_, _> = {
@@ -918,7 +887,7 @@ impl GlobalBarrierManagerContext {
 
             let (reschedule_fragment, applied_reschedules) = self
                 .scale_controller
-                .prepare_reschedule_command(
+                .analyze_reschedule_plan(
                     plan,
                     RescheduleOptions {
                         resolve_no_shuffle_upstream: true,
@@ -959,115 +928,155 @@ impl GlobalBarrierManagerContext {
     }
 
     /// This function will generate a migration plan, which includes the mapping for all expired and
-    /// in-used parallel unit to a new one.
+    /// in-used worker slot to a new one.
     async fn generate_migration_plan(
         &self,
         expired_workers: HashSet<WorkerId>,
         active_nodes: &mut ActiveStreamingWorkerNodes,
     ) -> MetaResult<MigrationPlan> {
         let mgr = self.metadata_manager.as_v1_ref();
-
         let mut cached_plan = MigrationPlan::get(self.env.meta_store().as_kv()).await?;
 
-        let all_worker_parallel_units = mgr.fragment_manager.all_worker_parallel_units().await;
+        let all_worker_slots = mgr.fragment_manager.all_worker_slots().await;
 
-        let (expired_inuse_workers, inuse_workers): (Vec<_>, Vec<_>) = all_worker_parallel_units
+        let (expired_inuse_workers, inuse_workers): (Vec<_>, Vec<_>) = all_worker_slots
             .into_iter()
             .partition(|(worker, _)| expired_workers.contains(worker));
 
-        let mut to_migrate_parallel_units: BTreeSet<_> = expired_inuse_workers
+        let mut to_migrate_worker_slots: BTreeSet<_> = expired_inuse_workers
             .into_iter()
-            .flat_map(|(_, pu)| pu.into_iter())
+            .flat_map(|(_, worker_slots)| worker_slots.into_iter())
             .collect();
-        let mut inuse_parallel_units: HashSet<_> = inuse_workers
+        let mut inuse_worker_slots: HashSet<_> = inuse_workers
             .into_iter()
-            .flat_map(|(_, pu)| pu.into_iter())
+            .flat_map(|(_, worker_slots)| worker_slots.into_iter())
             .collect();
 
-        cached_plan.parallel_unit_plan.retain(|from, to| {
-            if to_migrate_parallel_units.contains(from) {
-                if !to_migrate_parallel_units.contains(&to.id) {
-                    // clean up target parallel units in migration plan that are expired and not
+        cached_plan.worker_slot_plan.retain(|from, to| {
+            if to_migrate_worker_slots.contains(from) {
+                if !to_migrate_worker_slots.contains(to) {
+                    // clean up target worker slots in migration plan that are expired and not
                     // used by any actors.
-                    return !expired_workers.contains(&to.worker_node_id);
+                    return !expired_workers.contains(&to.worker_id());
                 }
                 return true;
             }
             false
         });
-        to_migrate_parallel_units.retain(|id| !cached_plan.parallel_unit_plan.contains_key(id));
-        inuse_parallel_units.extend(cached_plan.parallel_unit_plan.values().map(|pu| pu.id));
+        to_migrate_worker_slots.retain(|id| !cached_plan.worker_slot_plan.contains_key(id));
+        inuse_worker_slots.extend(cached_plan.worker_slot_plan.values());
 
-        if to_migrate_parallel_units.is_empty() {
-            // all expired parallel units are already in migration plan.
-            debug!("all expired parallel units are already in migration plan.");
+        if to_migrate_worker_slots.is_empty() {
+            // all expired worker slots are already in migration plan.
+            debug!("all expired worker slots are already in migration plan.");
             return Ok(cached_plan);
         }
-        let mut to_migrate_parallel_units =
-            to_migrate_parallel_units.into_iter().rev().collect_vec();
-        debug!(
-            "got to migrate parallel units {:#?}",
-            to_migrate_parallel_units
-        );
+        let mut to_migrate_worker_slots = to_migrate_worker_slots.into_iter().rev().collect_vec();
+        debug!("got to migrate worker slots {:#?}", to_migrate_worker_slots);
 
         let start = Instant::now();
-        // if in-used expire parallel units are not empty, should wait for newly joined worker.
-        'discovery: while !to_migrate_parallel_units.is_empty() {
-            let mut new_parallel_units = active_nodes
+        // if in-used expire worker slots are not empty, should wait for newly joined worker.
+        'discovery: while !to_migrate_worker_slots.is_empty() {
+            let mut new_worker_slots = active_nodes
                 .current()
                 .values()
-                .flat_map(|worker| worker.parallel_units.iter().cloned())
+                .flat_map(|worker| {
+                    (0..worker.parallelism).map(move |i| WorkerSlotId::new(worker.id, i as _))
+                })
                 .collect_vec();
-            new_parallel_units.retain(|pu| !inuse_parallel_units.contains(&pu.id));
 
-            if !new_parallel_units.is_empty() {
-                debug!("new parallel units found: {:#?}", new_parallel_units);
-                for target_parallel_unit in new_parallel_units {
-                    if let Some(from) = to_migrate_parallel_units.pop() {
+            new_worker_slots.retain(|worker_slot| !inuse_worker_slots.contains(worker_slot));
+
+            let to_migration_size = to_migrate_worker_slots.len();
+            let mut available_size = new_worker_slots.len();
+
+            if available_size < to_migration_size
+                && start.elapsed() > GlobalBarrierManager::RECOVERY_FORCE_MIGRATION_TIMEOUT
+            {
+                let mut factor = 2;
+                while available_size < to_migration_size {
+                    let mut extended_worker_slots = active_nodes
+                        .current()
+                        .values()
+                        .flat_map(|worker| {
+                            (0..worker.parallelism * factor)
+                                .map(move |i| WorkerSlotId::new(worker.id, i as _))
+                        })
+                        .collect_vec();
+
+                    extended_worker_slots
+                        .retain(|worker_slot| !inuse_worker_slots.contains(worker_slot));
+
+                    extended_worker_slots.sort_by(|a, b| {
+                        a.slot_idx()
+                            .cmp(&b.slot_idx())
+                            .then(a.worker_id().cmp(&b.worker_id()))
+                    });
+
+                    available_size = extended_worker_slots.len();
+                    new_worker_slots = extended_worker_slots;
+                    factor *= 2;
+                }
+
+                tracing::info!(
+                    "migration timed out, extending worker slots to {:?} by factor {}",
+                    new_worker_slots,
+                    factor,
+                );
+            }
+
+            if !new_worker_slots.is_empty() {
+                debug!("new worker slots found: {:#?}", new_worker_slots);
+                for target_worker_slot in new_worker_slots {
+                    if let Some(from) = to_migrate_worker_slots.pop() {
                         debug!(
-                            "plan to migrate from parallel unit {} to {}",
-                            from, target_parallel_unit.id
+                            "plan to migrate from worker slot {} to {}",
+                            from, target_worker_slot
                         );
-                        inuse_parallel_units.insert(target_parallel_unit.id);
+                        inuse_worker_slots.insert(target_worker_slot);
                         cached_plan
-                            .parallel_unit_plan
-                            .insert(from, target_parallel_unit);
+                            .worker_slot_plan
+                            .insert(from, target_worker_slot);
                     } else {
                         break 'discovery;
                     }
                 }
             }
 
-            if to_migrate_parallel_units.is_empty() {
+            if to_migrate_worker_slots.is_empty() {
                 break;
             }
 
             // wait to get newly joined CN
             let changed = active_nodes
-                .wait_changed(Duration::from_millis(5000), |active_nodes| {
-                    let current_nodes = active_nodes
-                        .current()
-                        .values()
-                        .map(|node| (node.id, &node.host, &node.parallel_units))
-                        .collect_vec();
-                    warn!(
-                        current_nodes = ?current_nodes,
-                        "waiting for new workers to join, elapsed: {}s",
-                        start.elapsed().as_secs()
-                    );
-                })
+                .wait_changed(
+                    Duration::from_millis(5000),
+                    GlobalBarrierManager::RECOVERY_FORCE_MIGRATION_TIMEOUT,
+                    |active_nodes| {
+                        let current_nodes = active_nodes
+                            .current()
+                            .values()
+                            .map(|node| (node.id, &node.host, &node.parallelism))
+                            .collect_vec();
+                        warn!(
+                            current_nodes = ?current_nodes,
+                            "waiting for new workers to join, elapsed: {}s",
+                            start.elapsed().as_secs()
+                        );
+                    },
+                )
                 .await;
-            warn!(?changed, "get worker changed. Retry migrate");
+            warn!(?changed, "get worker changed or timed out. Retry migrate");
         }
 
         // update migration plan, if there is a chain in the plan, update it.
         let mut new_plan = MigrationPlan::default();
-        for (from, to) in &cached_plan.parallel_unit_plan {
-            let mut to = to.clone();
-            while let Some(target) = cached_plan.parallel_unit_plan.get(&to.id) {
-                to = target.clone();
+        for (from, to) in &cached_plan.worker_slot_plan {
+            let mut to = *to;
+            while let Some(target) = cached_plan.worker_slot_plan.get(&to) {
+                to = *target;
             }
-            new_plan.parallel_unit_plan.insert(*from, to);
+            new_plan.worker_slot_plan.insert(*from, to);
         }
 
         new_plan.insert(self.env.meta_store().as_kv()).await?;
diff --git a/src/meta/src/barrier/rpc.rs b/src/meta/src/barrier/rpc.rs
index 0a7e6d4e1e950..ae12c439e0a93 100644
--- a/src/meta/src/barrier/rpc.rs
+++ b/src/meta/src/barrier/rpc.rs
@@ -12,10 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::{HashMap, HashSet, VecDeque};
+use std::collections::{HashMap, HashSet};
 use std::error::Error;
 use std::future::Future;
-use std::sync::Arc;
 use std::time::Duration;
 
 use anyhow::anyhow;
@@ -46,7 +45,9 @@ use uuid::Uuid;
 
 use super::command::CommandContext;
 use super::GlobalBarrierManagerContext;
-use crate::manager::{MetaSrvEnv, WorkerId};
+use crate::barrier::info::InflightActorInfo;
+use crate::manager::{InflightFragmentInfo, MetaSrvEnv, WorkerId};
+use crate::model::FragmentId;
 use crate::{MetaError, MetaResult};
 
 const COLLECT_ERROR_TIMEOUT: Duration = Duration::from_secs(3);
@@ -54,8 +55,6 @@ const COLLECT_ERROR_TIMEOUT: Duration = Duration::from_secs(3);
 struct ControlStreamNode {
     worker: WorkerNode,
     sender: UnboundedSender<StreamingControlStreamRequest>,
-    // earlier epoch at the front
-    inflight_barriers: VecDeque<Arc<CommandContext>>,
 }
 
 fn into_future(
@@ -166,62 +165,64 @@ impl ControlStreamManager {
         Ok(())
     }
 
+    /// Clear all nodes and response streams in the manager.
+    pub(super) fn clear(&mut self) {
+        *self = Self::new(self.context.clone());
+    }
+
     async fn next_response(
         &mut self,
     ) -> Option<(WorkerId, MetaResult<StreamingControlStreamResponse>)> {
         let (worker_id, response_stream, result) = self.response_streams.next().await?;
-        if result.is_ok() {
-            self.response_streams
-                .push(into_future(worker_id, response_stream));
+
+        match result.as_ref().map(|r| r.response.as_ref().unwrap()) {
+            Ok(streaming_control_stream_response::Response::Shutdown(_)) | Err(_) => {
+                // Do not add it back to the `response_streams` so that it will not be polled again.
+            }
+            _ => {
+                self.response_streams
+                    .push(into_future(worker_id, response_stream));
+            }
         }
+
         Some((worker_id, result))
     }
 
     pub(super) async fn next_complete_barrier_response(
         &mut self,
-    ) -> MetaResult<(WorkerId, u64, BarrierCompleteResponse)> {
-        loop {
+    ) -> (WorkerId, MetaResult<BarrierCompleteResponse>) {
+        use streaming_control_stream_response::Response;
+
+        {
             let (worker_id, result) = pending_on_none(self.next_response()).await;
-            match result {
-                Ok(resp) => match resp.response {
-                    Some(streaming_control_stream_response::Response::CompleteBarrier(resp)) => {
-                        let node = self
-                            .nodes
-                            .get_mut(&worker_id)
-                            .expect("should exist when get collect resp");
-                        let command = node
-                            .inflight_barriers
-                            .pop_front()
-                            .expect("should exist when get collect resp");
-                        break Ok((worker_id, command.prev_epoch.value().0, resp));
+            let result = match result {
+                Ok(resp) => match resp.response.unwrap() {
+                    Response::CompleteBarrier(resp) => {
+                        assert_eq!(worker_id, resp.worker_id);
+                        Ok(resp)
                     }
-                    resp => {
-                        break Err(anyhow!("get unexpected resp: {:?}", resp).into());
+                    Response::Shutdown(_) => {
+                        Err(anyhow!("worker node {worker_id} is shutting down").into())
                     }
-                },
-                Err(err) => {
-                    let mut node = self
-                        .nodes
-                        .remove(&worker_id)
-                        .expect("should exist when get collect resp");
-                    // Note: No need to use `?` as the backtrace is from meta and not useful.
-                    warn!(node = ?node.worker, err = %err.as_report(), "get error from response stream");
-                    if let Some(command) = node.inflight_barriers.pop_front() {
-                        let errors = self.collect_errors(node.worker.id, err).await;
-                        let err = merge_node_rpc_errors("get error from control stream", errors);
-                        self.context.report_collect_failure(&command, &err);
-                        break Err(err);
-                    } else {
-                        // for node with no inflight barrier, simply ignore the error
-                        info!(node = ?node.worker, "no inflight barrier no node. Ignore error");
-                        continue;
+                    Response::Init(_) => {
+                        // This arm should be unreachable.
+                        Err(anyhow!("get unexpected init response").into())
                     }
-                }
+                },
+                Err(err) => Err(err),
+            };
+            if let Err(err) = &result {
+                let node = self
+                    .nodes
+                    .remove(&worker_id)
+                    .expect("should exist when get shutdown resp");
+                warn!(node = ?node.worker, err = %err.as_report(), "get error from response stream");
             }
+            (worker_id, result)
         }
     }
 
-    async fn collect_errors(
+    pub(super) async fn collect_errors(
         &mut self,
         worker_id: WorkerId,
         first_err: MetaError,
@@ -238,6 +239,7 @@ impl ControlStreamManager {
             })
             .await;
         }
+        tracing::debug!(?errors, "collected stream errors");
         errors
     }
 }
@@ -246,7 +248,9 @@ impl ControlStreamManager {
     /// Send inject-barrier-rpc to stream service and wait for its response before returns.
     pub(super) fn inject_barrier(
         &mut self,
-        command_context: Arc<CommandContext>,
+        command_context: &CommandContext,
+        pre_applied_fragment_infos: &HashMap<FragmentId, InflightFragmentInfo>,
+        applied_fragment_infos: Option<&HashMap<FragmentId, InflightFragmentInfo>>,
     ) -> MetaResult<HashSet<WorkerId>> {
         fail_point!("inject_barrier_err", |_| risingwave_common::bail!(
             "inject_barrier_err"
@@ -258,14 +262,30 @@ impl ControlStreamManager {
         info.node_map
             .iter()
             .map(|(node_id, worker_node)| {
-                let actor_ids_to_send = info.actor_ids_to_send(node_id).collect_vec();
-                let actor_ids_to_collect = info.actor_ids_to_collect(node_id).collect_vec();
+                let actor_ids_to_send: Vec<_> =
+                    InflightActorInfo::actor_ids_to_send(pre_applied_fragment_infos, *node_id)
+                        .collect();
+                let actor_ids_to_collect: Vec<_> =
+                    InflightActorInfo::actor_ids_to_collect(pre_applied_fragment_infos, *node_id)
+                        .collect();
                 if actor_ids_to_collect.is_empty() {
                     // No need to send or collect barrier for this node.
                     assert!(actor_ids_to_send.is_empty());
-                    Ok(())
+                }
+                let table_ids_to_sync = if let Some(fragment_infos) = applied_fragment_infos {
+                    InflightActorInfo::existing_table_ids(fragment_infos)
+                        .map(|table_id| table_id.table_id)
+                        .collect()
                 } else {
+                    Default::default()
+                };
+
+                {
                     let Some(node) = self.nodes.get_mut(node_id) else {
+                        if actor_ids_to_collect.is_empty() {
+                            // Worker node get disconnected but has no actor to collect. Simply skip it.
+                            return Ok(());
+                        }
                         return Err(
                             anyhow!("unconnected worker node: {:?}", worker_node.host).into()
                         );
@@ -294,12 +314,8 @@ impl ControlStreamManager {
                                         barrier: Some(barrier),
                                         actor_ids_to_send,
                                         actor_ids_to_collect,
-                                        table_ids_to_sync: command_context
-                                            .info
-                                            .existing_table_ids()
-                                            .iter()
-                                            .map(|table_id| table_id.table_id)
-                                            .collect(),
+                                        table_ids_to_sync,
+                                        partial_graph_id: u32::MAX,
                                     },
                                 ),
                             ),
@@ -312,7 +328,6 @@ impl ControlStreamManager {
                             ))
                         })?;
 
-                    node.inflight_barriers.push_back(command_context.clone());
                     node_need_collect.insert(*node_id);
                     Result::<_, MetaError>::Ok(())
                 }
@@ -355,14 +370,17 @@ impl GlobalBarrierManagerContext {
             ControlStreamNode {
                 worker: node.clone(),
                 sender: handle.request_sender,
-                inflight_barriers: VecDeque::new(),
             },
             handle.response_stream,
         ))
     }
 
     /// Send barrier-complete-rpc and wait for responses from all CNs
-    fn report_collect_failure(&self, command_context: &CommandContext, error: &MetaError) {
+    pub(super) fn report_collect_failure(
+        &self,
+        command_context: &CommandContext,
+        error: &MetaError,
+    ) {
         // Record failure in event log.
         use risingwave_pb::meta::event_log;
         let event = event_log::EventCollectBarrierFail {
@@ -532,17 +550,54 @@ where
     Err(results_err)
 }
 
-fn merge_node_rpc_errors<E: Error>(
+pub(super) fn merge_node_rpc_errors<E: Error + Send + Sync + 'static>(
     message: &str,
     errors: impl IntoIterator<Item = (WorkerId, E)>,
 ) -> MetaError {
+    use std::error::request_value;
     use std::fmt::Write;
 
+    use risingwave_common::error::tonic::extra::Score;
+
+    let errors = errors.into_iter().collect_vec();
+
+    if errors.is_empty() {
+        return anyhow!(message.to_owned()).into();
+    }
+
+    // Create the error from the single error.
+    let single_error = |(worker_id, e)| {
+        anyhow::Error::from(e)
+            .context(format!("{message}, in worker node {worker_id}"))
+            .into()
+    };
+
+    if errors.len() == 1 {
+        return single_error(errors.into_iter().next().unwrap());
+    }
+
+    // Find the error with the highest score.
+    let max_score = errors
+        .iter()
+        .filter_map(|(_, e)| request_value::<Score>(e))
+        .max();
+
+    if let Some(max_score) = max_score {
+        let mut errors = errors;
+        let max_scored = errors
+            .extract_if(|(_, e)| request_value::<Score>(e) == Some(max_score))
+            .next()
+            .unwrap();
+
+        return single_error(max_scored);
+    }
+
+    // The errors do not have scores, so simply concatenate them.
     let concat: String = errors
         .into_iter()
-        .fold(format!("{message}:"), |mut s, (w, e)| {
-            write!(&mut s, " worker node {}, {};", w, e.as_report()).unwrap();
+        .fold(format!("{message}: "), |mut s, (w, e)| {
+            write!(&mut s, " in worker node {}, {};", w, e.as_report()).unwrap();
             s
         });
-    anyhow::anyhow!(concat).into()
+    anyhow!(concat).into()
 }
diff --git a/src/meta/src/barrier/schedule.rs b/src/meta/src/barrier/schedule.rs
index 50034811a3390..d2a2febbec2fa 100644
--- a/src/meta/src/barrier/schedule.rs
+++ b/src/meta/src/barrier/schedule.rs
@@ -184,10 +184,8 @@ impl BarrierScheduler {
     pub fn try_cancel_scheduled_create(&self, table_id: TableId) -> bool {
         let queue = &mut self.inner.queue.lock();
         if let Some(idx) = queue.queue.iter().position(|scheduled| {
-            if let Command::CreateStreamingJob {
-                table_fragments, ..
-            } = &scheduled.command
-                && table_fragments.table_id() == table_id
+            if let Command::CreateStreamingJob { info, .. } = &scheduled.command
+                && info.table_fragments.table_id() == table_id
             {
                 true
             } else {
@@ -259,16 +257,14 @@ impl BarrierScheduler {
         for command in commands {
             let (started_tx, started_rx) = oneshot::channel();
             let (collect_tx, collect_rx) = oneshot::channel();
-            let (finish_tx, finish_rx) = oneshot::channel();
 
-            contexts.push((started_rx, collect_rx, finish_rx));
+            contexts.push((started_rx, collect_rx));
             scheduleds.push(self.inner.new_scheduled(
                 command.need_checkpoint(),
                 command,
                 once(Notifier {
                     started: Some(started_tx),
                     collected: Some(collect_tx),
-                    finished: Some(finish_tx),
                 }),
             ));
         }
@@ -277,7 +273,7 @@ impl BarrierScheduler {
 
         let mut infos = Vec::with_capacity(contexts.len());
 
-        for (injected_rx, collect_rx, finish_rx) in contexts {
+        for (injected_rx, collect_rx) in contexts {
             // Wait for this command to be injected, and record the result.
             tracing::trace!("waiting for injected_rx");
             let info = injected_rx.await.ok().context("failed to inject barrier")?;
@@ -289,10 +285,6 @@ impl BarrierScheduler {
                 .await
                 .ok()
                 .context("failed to collect barrier")??;
-
-            tracing::trace!("waiting for finish_rx");
-            // Wait for this command to be finished.
-            finish_rx.await.ok().context("failed to finish command")??;
         }
 
         Ok(infos)
@@ -448,9 +440,8 @@ impl ScheduledBarriers {
                     unreachable!("only drop and cancel streaming jobs should be buffered");
                 }
             }
-            notifiers.into_iter().for_each(|mut notify| {
+            notifiers.into_iter().for_each(|notify| {
                 notify.notify_collected();
-                notify.notify_finished();
             });
         }
         (dropped_actors, cancel_table_ids)
diff --git a/src/meta/src/controller/catalog.rs b/src/meta/src/controller/catalog.rs
index a49fc7a8a0690..6303262944b0e 100644
--- a/src/meta/src/controller/catalog.rs
+++ b/src/meta/src/controller/catalog.rs
@@ -14,11 +14,13 @@
 
 use std::collections::{BTreeSet, HashMap, HashSet};
 use std::iter;
+use std::mem::take;
 use std::sync::Arc;
 
 use anyhow::anyhow;
 use itertools::Itertools;
 use risingwave_common::catalog::{TableOption, DEFAULT_SCHEMA_NAME, SYSTEM_SCHEMAS};
+use risingwave_common::secret::LocalSecretManager;
 use risingwave_common::util::stream_graph_visitor::visit_stream_node_cont_mut;
 use risingwave_common::{bail, current_cluster_version};
 use risingwave_connector::source::UPSTREAM_SOURCE_KEY;
@@ -45,9 +47,7 @@ use risingwave_pb::meta::relation::PbRelationInfo;
 use risingwave_pb::meta::subscribe_response::{
     Info as NotificationInfo, Info, Operation as NotificationOperation, Operation,
 };
-use risingwave_pb::meta::{
-    FragmentParallelUnitMapping, PbFragmentWorkerSlotMapping, PbRelation, PbRelationGroup,
-};
+use risingwave_pb::meta::{PbFragmentWorkerSlotMapping, PbRelation, PbRelationGroup};
 use risingwave_pb::stream_plan::stream_node::NodeBody;
 use risingwave_pb::stream_plan::FragmentTypeFlag;
 use risingwave_pb::user::PbUserInfo;
@@ -58,17 +58,17 @@ use sea_orm::{
     IntoActiveModel, JoinType, PaginatorTrait, QueryFilter, QuerySelect, RelationTrait,
     TransactionTrait, Value,
 };
-use tokio::sync::{RwLock, RwLockReadGuard};
+use tokio::sync::oneshot::Sender;
+use tokio::sync::{RwLock, RwLockReadGuard, RwLockWriteGuard};
 
-use super::utils::check_subscription_name_duplicate;
+use super::utils::{check_subscription_name_duplicate, get_fragment_ids_by_jobs};
 use crate::controller::rename::{alter_relation_rename, alter_relation_rename_refs};
 use crate::controller::utils::{
-    check_connection_name_duplicate, check_database_name_duplicate,
+    build_relation_group, check_connection_name_duplicate, check_database_name_duplicate,
     check_function_signature_duplicate, check_relation_name_duplicate, check_schema_name_duplicate,
     check_secret_name_duplicate, ensure_object_id, ensure_object_not_refer, ensure_schema_empty,
-    ensure_user_id, get_fragment_mappings_by_jobs, get_referring_objects,
-    get_referring_objects_cascade, get_user_privilege, list_user_info_by_ids,
-    resolve_source_register_info_for_jobs, PartialObject,
+    ensure_user_id, get_referring_objects, get_referring_objects_cascade, get_user_privilege,
+    list_user_info_by_ids, resolve_source_register_info_for_jobs, PartialObject,
 };
 use crate::controller::ObjectModel;
 use crate::manager::{Catalog, MetaSrvEnv, NotificationVersion, IGNORED_NOTIFICATION_VERSION};
@@ -110,6 +110,7 @@ impl CatalogController {
             env,
             inner: RwLock::new(CatalogControllerInner {
                 db: meta_store.conn,
+                creating_table_finish_notifier: HashMap::new(),
             }),
         }
     }
@@ -119,10 +120,20 @@ impl CatalogController {
     pub async fn get_inner_read_guard(&self) -> RwLockReadGuard<'_, CatalogControllerInner> {
         self.inner.read().await
     }
+
+    pub async fn get_inner_write_guard(&self) -> RwLockWriteGuard<'_, CatalogControllerInner> {
+        self.inner.write().await
+    }
 }
 
 pub struct CatalogControllerInner {
     pub(crate) db: DatabaseConnection,
+    /// Registered finish notifiers for creating tables.
+    ///
+    /// `DdlController` will update this map, and pass the `tx` side to `CatalogController`.
+    /// On notifying, we can remove the entry from this map.
+    pub creating_table_finish_notifier:
+        HashMap<ObjectId, Vec<Sender<MetaResult<NotificationVersion>>>>,
 }
 
 impl CatalogController {
@@ -147,6 +158,10 @@ impl CatalogController {
             .notify_frontend_relation_info(operation, relation_info)
             .await
     }
+
+    pub(crate) async fn current_notification_version(&self) -> NotificationVersion {
+        self.env.notification_manager().current_version().await
+    }
 }
 
 impl CatalogController {
@@ -280,15 +295,13 @@ impl CatalogController {
             .all(&txn)
             .await?;
 
-        let fragment_mappings = get_fragment_mappings_by_jobs(&txn, streaming_jobs.clone())
+        let fragment_mappings = get_fragment_ids_by_jobs(&txn, streaming_jobs.clone())
             .await?
             .into_iter()
-            .map(
-                |FragmentParallelUnitMapping { fragment_id, .. }| PbFragmentWorkerSlotMapping {
-                    fragment_id,
-                    mapping: None,
-                },
-            )
+            .map(|fragment_id| PbFragmentWorkerSlotMapping {
+                fragment_id: fragment_id as _,
+                mapping: None,
+            })
             .collect();
 
         // The schema and objects in the database will be delete cascade.
@@ -630,15 +643,19 @@ impl CatalogController {
     }
 
     /// `clean_dirty_creating_jobs` cleans up creating jobs that are creating in Foreground mode or in Initial status.
-    /// FIXME(kwannoel): Notify deleted objects to the frontend.
     pub async fn clean_dirty_creating_jobs(&self) -> MetaResult<ReleaseContext> {
         let inner = self.inner.write().await;
         let txn = inner.db.begin().await?;
 
-        let creating_jobs: Vec<(ObjectId, ObjectType)> = streaming_job::Entity::find()
+        let mut dirty_objs: Vec<PartialObject> = streaming_job::Entity::find()
             .select_only()
             .column(streaming_job::Column::JobId)
-            .column(object::Column::ObjType)
+            .columns([
+                object::Column::Oid,
+                object::Column::ObjType,
+                object::Column::SchemaId,
+                object::Column::DatabaseId,
+            ])
             .join(JoinType::InnerJoin, streaming_job::Relation::Object.def())
             .filter(
                 streaming_job::Column::JobStatus.eq(JobStatus::Initial).or(
@@ -647,13 +664,13 @@ impl CatalogController {
                         .and(streaming_job::Column::CreateType.eq(CreateType::Foreground)),
                 ),
             )
-            .into_tuple()
+            .into_partial_model()
             .all(&txn)
             .await?;
 
         let changed = Self::clean_dirty_sink_downstreams(&txn).await?;
 
-        if creating_jobs.is_empty() {
+        if dirty_objs.is_empty() {
             if changed {
                 txn.commit().await?;
             }
@@ -661,22 +678,114 @@ impl CatalogController {
             return Ok(ReleaseContext::default());
         }
 
+        self.log_cleaned_dirty_jobs(&dirty_objs, &txn).await?;
+
+        let dirty_job_ids = dirty_objs.iter().map(|obj| obj.oid).collect::<Vec<_>>();
+
+        // Filter out dummy objs for replacement.
+        // todo: we'd better introduce a new dummy object type for replacement.
+        let all_dirty_table_ids = dirty_objs
+            .iter()
+            .filter(|obj| obj.obj_type == ObjectType::Table)
+            .map(|obj| obj.oid)
+            .collect_vec();
+        let dirty_table_ids: HashSet<ObjectId> = Table::find()
+            .select_only()
+            .column(table::Column::TableId)
+            .filter(table::Column::TableId.is_in(all_dirty_table_ids))
+            .into_tuple::<ObjectId>()
+            .all(&txn)
+            .await?
+            .into_iter()
+            .collect();
+        dirty_objs
+            .retain(|obj| obj.obj_type != ObjectType::Table || dirty_table_ids.contains(&obj.oid));
+
+        let associated_source_ids: Vec<SourceId> = Table::find()
+            .select_only()
+            .column(table::Column::OptionalAssociatedSourceId)
+            .filter(
+                table::Column::TableId
+                    .is_in(dirty_job_ids.clone())
+                    .and(table::Column::OptionalAssociatedSourceId.is_not_null()),
+            )
+            .into_tuple()
+            .all(&txn)
+            .await?;
+        let dirty_source_objs: Vec<PartialObject> = Object::find()
+            .filter(object::Column::Oid.is_in(associated_source_ids.clone()))
+            .into_partial_model()
+            .all(&txn)
+            .await?;
+        dirty_objs.extend(dirty_source_objs);
+
+        let mut dirty_state_table_ids = vec![];
+        let to_drop_internal_table_objs: Vec<PartialObject> = Object::find()
+            .select_only()
+            .columns([
+                object::Column::Oid,
+                object::Column::ObjType,
+                object::Column::SchemaId,
+                object::Column::DatabaseId,
+            ])
+            .join(JoinType::InnerJoin, object::Relation::Table.def())
+            .filter(table::Column::BelongsToJobId.is_in(dirty_job_ids.clone()))
+            .into_partial_model()
+            .all(&txn)
+            .await?;
+        dirty_state_table_ids.extend(to_drop_internal_table_objs.iter().map(|obj| obj.oid));
+        dirty_objs.extend(to_drop_internal_table_objs);
+
+        let to_delete_objs: HashSet<ObjectId> = dirty_job_ids
+            .clone()
+            .into_iter()
+            .chain(dirty_state_table_ids.clone().into_iter())
+            .chain(associated_source_ids.clone().into_iter())
+            .collect();
+
+        let res = Object::delete_many()
+            .filter(object::Column::Oid.is_in(to_delete_objs))
+            .exec(&txn)
+            .await?;
+        assert!(res.rows_affected > 0);
+
+        txn.commit().await?;
+
+        let relation_group = build_relation_group(dirty_objs);
+
+        let _version = self
+            .notify_frontend(NotificationOperation::Delete, relation_group)
+            .await;
+
+        Ok(ReleaseContext {
+            state_table_ids: dirty_state_table_ids,
+            source_ids: associated_source_ids,
+            ..Default::default()
+        })
+    }
+
+    async fn log_cleaned_dirty_jobs(
+        &self,
+        dirty_objs: &[PartialObject],
+        txn: &DatabaseTransaction,
+    ) -> MetaResult<()> {
         // Record cleaned streaming jobs in event logs.
-        let mut creating_table_ids = vec![];
-        let mut creating_source_ids = vec![];
-        let mut creating_sink_ids = vec![];
-        let mut creating_job_ids = vec![];
-        for (job_id, job_type) in creating_jobs {
-            creating_job_ids.push(job_id);
+        let mut dirty_table_ids = vec![];
+        let mut dirty_source_ids = vec![];
+        let mut dirty_sink_ids = vec![];
+        for dirty_job_obj in dirty_objs {
+            let job_id = dirty_job_obj.oid;
+            let job_type = dirty_job_obj.obj_type;
             match job_type {
-                ObjectType::Table | ObjectType::Index => creating_table_ids.push(job_id),
-                ObjectType::Source => creating_source_ids.push(job_id),
-                ObjectType::Sink => creating_sink_ids.push(job_id),
+                ObjectType::Table | ObjectType::Index => dirty_table_ids.push(job_id),
+                ObjectType::Source => dirty_source_ids.push(job_id),
+                ObjectType::Sink => dirty_sink_ids.push(job_id),
                 _ => unreachable!("unexpected streaming job type"),
             }
         }
+
         let mut event_logs = vec![];
-        if !creating_table_ids.is_empty() {
+        if !dirty_table_ids.is_empty() {
             let table_info: Vec<(TableId, String, String)> = Table::find()
                 .select_only()
                 .columns([
@@ -684,9 +793,9 @@ impl CatalogController {
                     table::Column::Name,
                     table::Column::Definition,
                 ])
-                .filter(table::Column::TableId.is_in(creating_table_ids))
+                .filter(table::Column::TableId.is_in(dirty_table_ids))
                 .into_tuple()
-                .all(&txn)
+                .all(txn)
                 .await?;
             for (table_id, name, definition) in table_info {
                 let event = risingwave_pb::meta::event_log::EventDirtyStreamJobClear {
@@ -700,7 +809,7 @@ impl CatalogController {
                 ));
             }
         }
-        if !creating_source_ids.is_empty() {
+        if !dirty_source_ids.is_empty() {
             let source_info: Vec<(SourceId, String, String)> = Source::find()
                 .select_only()
                 .columns([
@@ -708,9 +817,9 @@ impl CatalogController {
                     source::Column::Name,
                     source::Column::Definition,
                 ])
-                .filter(source::Column::SourceId.is_in(creating_source_ids))
+                .filter(source::Column::SourceId.is_in(dirty_source_ids))
                 .into_tuple()
-                .all(&txn)
+                .all(txn)
                 .await?;
             for (source_id, name, definition) in source_info {
                 let event = risingwave_pb::meta::event_log::EventDirtyStreamJobClear {
@@ -724,7 +833,7 @@ impl CatalogController {
                 ));
             }
         }
-        if !creating_sink_ids.is_empty() {
+        if !dirty_sink_ids.is_empty() {
             let sink_info: Vec<(SinkId, String, String)> = Sink::find()
                 .select_only()
                 .columns([
@@ -732,9 +841,9 @@ impl CatalogController {
                     sink::Column::Name,
                     sink::Column::Definition,
                 ])
-                .filter(sink::Column::SinkId.is_in(creating_sink_ids))
+                .filter(sink::Column::SinkId.is_in(dirty_sink_ids))
                 .into_tuple()
-                .all(&txn)
+                .all(txn)
                 .await?;
             for (sink_id, name, definition) in sink_info {
                 let event = risingwave_pb::meta::event_log::EventDirtyStreamJobClear {
@@ -748,53 +857,8 @@ impl CatalogController {
                 ));
             }
         }
-
-        let associated_source_ids: Vec<SourceId> = Table::find()
-            .select_only()
-            .column(table::Column::OptionalAssociatedSourceId)
-            .filter(
-                table::Column::TableId
-                    .is_in(creating_job_ids.clone())
-                    .and(table::Column::OptionalAssociatedSourceId.is_not_null()),
-            )
-            .into_tuple()
-            .all(&txn)
-            .await?;
-
-        let state_table_ids: Vec<TableId> = Table::find()
-            .select_only()
-            .column(table::Column::TableId)
-            .filter(
-                table::Column::BelongsToJobId
-                    .is_in(creating_job_ids.clone())
-                    .or(table::Column::TableId.is_in(creating_job_ids.clone())),
-            )
-            .into_tuple()
-            .all(&txn)
-            .await?;
-
-        let to_delete_objs: HashSet<ObjectId> = creating_job_ids
-            .clone()
-            .into_iter()
-            .chain(state_table_ids.clone().into_iter())
-            .chain(associated_source_ids.clone().into_iter())
-            .collect();
-
-        let res = Object::delete_many()
-            .filter(object::Column::Oid.is_in(to_delete_objs))
-            .exec(&txn)
-            .await?;
-        assert!(res.rows_affected > 0);
-
-        txn.commit().await?;
-
         self.env.event_log_manager_ref().add_event_logs(event_logs);
-
-        Ok(ReleaseContext {
-            state_table_ids,
-            source_ids: associated_source_ids,
-            ..Default::default()
-        })
+        Ok(())
     }
 
     async fn clean_dirty_sink_downstreams(txn: &DatabaseTransaction) -> MetaResult<bool> {
@@ -1091,7 +1155,11 @@ impl CatalogController {
         Ok(version)
     }
 
-    pub async fn create_secret(&self, mut pb_secret: PbSecret) -> MetaResult<NotificationVersion> {
+    pub async fn create_secret(
+        &self,
+        mut pb_secret: PbSecret,
+        secret_plain_payload: Vec<u8>,
+    ) -> MetaResult<NotificationVersion> {
         let inner = self.inner.write().await;
         let owner_id = pb_secret.owner as _;
         let txn = inner.db.begin().await?;
@@ -1114,12 +1182,22 @@ impl CatalogController {
 
         txn.commit().await?;
 
+        // Notify the compute and frontend node plain secret
+        let mut secret_plain = pb_secret;
+        secret_plain.value.clone_from(&secret_plain_payload);
+
+        LocalSecretManager::global().add_secret(secret_plain.id, secret_plain_payload);
+        self.env
+            .notification_manager()
+            .notify_compute_without_version(Operation::Add, Info::Secret(secret_plain.clone()));
+
         let version = self
             .notify_frontend(
                 NotificationOperation::Add,
-                NotificationInfo::Secret(pb_secret),
+                NotificationInfo::Secret(secret_plain),
             )
             .await;
+
         Ok(version)
     }
 
@@ -1164,6 +1242,11 @@ impl CatalogController {
         let pb_secret: PbSecret = ObjectModel(secret, secret_obj.unwrap()).into();
 
         self.notify_users_update(user_infos).await;
+
+        LocalSecretManager::global().remove_secret(pb_secret.id);
+        self.env
+            .notification_manager()
+            .notify_compute_without_version(Operation::Delete, Info::Secret(pb_secret.clone()));
         let version = self
             .notify_frontend(
                 NotificationOperation::Delete,
@@ -2073,8 +2156,7 @@ impl CatalogController {
         let (source_fragments, removed_actors, removed_fragments) =
             resolve_source_register_info_for_jobs(&txn, to_drop_streaming_jobs.clone()).await?;
 
-        let fragment_mappings =
-            get_fragment_mappings_by_jobs(&txn, to_drop_streaming_jobs.clone()).await?;
+        let fragment_ids = get_fragment_ids_by_jobs(&txn, to_drop_streaming_jobs.clone()).await?;
 
         // Find affect users with privileges on all this objects.
         let to_update_user_ids: Vec<UserId> = UserPrivilege::find()
@@ -2103,85 +2185,18 @@ impl CatalogController {
 
         // notify about them.
         self.notify_users_update(user_infos).await;
-        let mut relations = vec![];
-        for obj in to_drop_objects {
-            match obj.obj_type {
-                ObjectType::Table => relations.push(PbRelation {
-                    relation_info: Some(PbRelationInfo::Table(PbTable {
-                        id: obj.oid as _,
-                        schema_id: obj.schema_id.unwrap() as _,
-                        database_id: obj.database_id.unwrap() as _,
-                        ..Default::default()
-                    })),
-                }),
-                ObjectType::Source => relations.push(PbRelation {
-                    relation_info: Some(PbRelationInfo::Source(PbSource {
-                        id: obj.oid as _,
-                        schema_id: obj.schema_id.unwrap() as _,
-                        database_id: obj.database_id.unwrap() as _,
-                        ..Default::default()
-                    })),
-                }),
-                ObjectType::Sink => relations.push(PbRelation {
-                    relation_info: Some(PbRelationInfo::Sink(PbSink {
-                        id: obj.oid as _,
-                        schema_id: obj.schema_id.unwrap() as _,
-                        database_id: obj.database_id.unwrap() as _,
-                        ..Default::default()
-                    })),
-                }),
-                ObjectType::Subscription => relations.push(PbRelation {
-                    relation_info: Some(PbRelationInfo::Subscription(PbSubscription {
-                        id: obj.oid as _,
-                        schema_id: obj.schema_id.unwrap() as _,
-                        database_id: obj.database_id.unwrap() as _,
-                        ..Default::default()
-                    })),
-                }),
-                ObjectType::View => relations.push(PbRelation {
-                    relation_info: Some(PbRelationInfo::View(PbView {
-                        id: obj.oid as _,
-                        schema_id: obj.schema_id.unwrap() as _,
-                        database_id: obj.database_id.unwrap() as _,
-                        ..Default::default()
-                    })),
-                }),
-                ObjectType::Index => {
-                    relations.push(PbRelation {
-                        relation_info: Some(PbRelationInfo::Index(PbIndex {
-                            id: obj.oid as _,
-                            schema_id: obj.schema_id.unwrap() as _,
-                            database_id: obj.database_id.unwrap() as _,
-                            ..Default::default()
-                        })),
-                    });
-                    relations.push(PbRelation {
-                        relation_info: Some(PbRelationInfo::Table(PbTable {
-                            id: obj.oid as _,
-                            schema_id: obj.schema_id.unwrap() as _,
-                            database_id: obj.database_id.unwrap() as _,
-                            ..Default::default()
-                        })),
-                    });
-                }
-                _ => unreachable!("only relations will be dropped."),
-            }
-        }
+        let relation_group = build_relation_group(to_drop_objects);
+
         let version = self
-            .notify_frontend(
-                NotificationOperation::Delete,
-                NotificationInfo::RelationGroup(PbRelationGroup { relations }),
-            )
+            .notify_frontend(NotificationOperation::Delete, relation_group)
             .await;
 
-        let fragment_mappings = fragment_mappings
+        let fragment_mappings = fragment_ids
             .into_iter()
-            .map(
-                |FragmentParallelUnitMapping { fragment_id, .. }| PbFragmentWorkerSlotMapping {
-                    fragment_id,
-                    mapping: None,
-                },
-            )
+            .map(|fragment_id| PbFragmentWorkerSlotMapping {
+                fragment_id: fragment_id as _,
+                mapping: None,
+            })
             .collect();
 
         self.notify_fragment_mapping(NotificationOperation::Delete, fragment_mappings)
@@ -2659,6 +2674,19 @@ impl CatalogController {
             .collect())
     }
 
+    pub async fn get_sink_by_ids(&self, sink_ids: Vec<SinkId>) -> MetaResult<Vec<PbSink>> {
+        let inner = self.inner.read().await;
+        let sink_objs = Sink::find()
+            .find_also_related(Object)
+            .filter(sink::Column::SinkId.is_in(sink_ids))
+            .all(&inner.db)
+            .await?;
+        Ok(sink_objs
+            .into_iter()
+            .map(|(sink, obj)| ObjectModel(sink, obj.unwrap()).into())
+            .collect())
+    }
+
     pub async fn get_subscription_by_id(
         &self,
         subscription_id: SubscriptionId,
@@ -2673,11 +2701,35 @@ impl CatalogController {
             .into_iter()
             .map(|(subscription, obj)| ObjectModel(subscription, obj.unwrap()).into())
             .find_or_first(|_| true)
-            .ok_or_else(|| anyhow!("cant find subscription with id {}", subscription_id))?;
+            .ok_or_else(|| anyhow!("cannot find subscription with id {}", subscription_id))?;
 
         Ok(subscription)
     }
 
+    pub async fn get_mv_depended_subscriptions(
+        &self,
+    ) -> MetaResult<HashMap<risingwave_common::catalog::TableId, HashMap<u32, u64>>> {
+        let inner = self.inner.read().await;
+        let subscription_objs = Subscription::find()
+            .find_also_related(Object)
+            .all(&inner.db)
+            .await?;
+        let mut map = HashMap::new();
+        // Write object at the same time we write subscription, so we must be able to get obj
+        for subscription in subscription_objs
+            .into_iter()
+            .map(|(subscription, obj)| ObjectModel(subscription, obj.unwrap()).into())
+        {
+            let subscription: PbSubscription = subscription;
+            map.entry(risingwave_common::catalog::TableId::from(
+                subscription.dependent_table_id,
+            ))
+            .or_insert(HashMap::new())
+            .insert(subscription.id, subscription.retention_seconds);
+        }
+        Ok(map)
+    }
+
     pub async fn find_creating_streaming_job_ids(
         &self,
         infos: Vec<PbCreatingJobInfo>,
@@ -3102,7 +3154,7 @@ impl CatalogControllerInner {
             .collect())
     }
 
-    async fn list_secrets(&self) -> MetaResult<Vec<PbSecret>> {
+    pub async fn list_secrets(&self) -> MetaResult<Vec<PbSecret>> {
         let secret_objs = Secret::find()
             .find_also_related(Object)
             .all(&self.db)
@@ -3124,6 +3176,42 @@ impl CatalogControllerInner {
             .map(|(func, obj)| ObjectModel(func, obj.unwrap()).into())
             .collect())
     }
+
+    pub(crate) fn register_finish_notifier(
+        &mut self,
+        id: i32,
+        sender: Sender<MetaResult<NotificationVersion>>,
+    ) {
+        self.creating_table_finish_notifier
+            .entry(id)
+            .or_default()
+            .push(sender);
+    }
+
+    pub(crate) async fn streaming_job_is_finished(&mut self, id: i32) -> MetaResult<bool> {
+        let status = StreamingJob::find()
+            .select_only()
+            .column(streaming_job::Column::JobStatus)
+            .filter(streaming_job::Column::JobId.eq(id))
+            .into_tuple::<JobStatus>()
+            .one(&self.db)
+            .await?;
+
+        status
+            .map(|status| status == JobStatus::Created)
+            .ok_or_else(|| {
+                MetaError::catalog_id_not_found("streaming job", "may have been cancelled/dropped")
+            })
+    }
+
+    pub(crate) fn notify_finish_failed(&mut self, err: &MetaError) {
+        for tx in take(&mut self.creating_table_finish_notifier)
+            .into_values()
+            .flatten()
+        {
+            let _ = tx.send(Err(err.clone()));
+        }
+    }
 }
 
 #[cfg(test)]
diff --git a/src/meta/src/controller/cluster.rs b/src/meta/src/controller/cluster.rs
index 50300f0ec2827..18643bc8c43c1 100644
--- a/src/meta/src/controller/cluster.rs
+++ b/src/meta/src/controller/cluster.rs
@@ -20,6 +20,7 @@ use std::sync::Arc;
 use std::time::{Duration, SystemTime};
 
 use itertools::Itertools;
+use risingwave_common::hash::WorkerSlotId;
 use risingwave_common::util::addr::HostAddr;
 use risingwave_common::util::resource_util::cpu::total_cpu_available;
 use risingwave_common::util::resource_util::memory::system_memory_available_bytes;
@@ -27,12 +28,9 @@ use risingwave_common::RW_VERSION;
 use risingwave_hummock_sdk::HummockSstableObjectId;
 use risingwave_meta_model_v2::prelude::{Worker, WorkerProperty};
 use risingwave_meta_model_v2::worker::{WorkerStatus, WorkerType};
-use risingwave_meta_model_v2::{worker, worker_property, I32Array, TransactionId, WorkerId};
+use risingwave_meta_model_v2::{worker, worker_property, TransactionId, WorkerId};
 use risingwave_pb::common::worker_node::{PbProperty, PbResource, PbState};
-use risingwave_pb::common::{
-    HostAddress, ParallelUnit, PbHostAddress, PbParallelUnit, PbWorkerNode, PbWorkerType,
-    WorkerNode,
-};
+use risingwave_pb::common::{HostAddress, PbHostAddress, PbWorkerNode, PbWorkerType, WorkerNode};
 use risingwave_pb::meta::add_worker_node_request::Property as AddNodeProperty;
 use risingwave_pb::meta::heartbeat_request;
 use risingwave_pb::meta::subscribe_response::{Info, Operation};
@@ -52,6 +50,7 @@ use tokio::task::JoinHandle;
 use crate::manager::{
     LocalNotification, MetaSrvEnv, StreamingClusterInfo, WorkerKey, META_NODE_ID,
 };
+use crate::model::ClusterId;
 use crate::{MetaError, MetaResult};
 
 pub type ClusterControllerRef = Arc<ClusterController>;
@@ -80,20 +79,7 @@ impl From<WorkerInfo> for PbWorkerNode {
                 port: info.0.port,
             }),
             state: PbState::from(info.0.status) as _,
-            parallel_units: info
-                .1
-                .as_ref()
-                .map(|p| {
-                    p.parallel_unit_ids
-                        .0
-                        .iter()
-                        .map(|&id| PbParallelUnit {
-                            id: id as _,
-                            worker_node_id: info.0.worker_id as _,
-                        })
-                        .collect_vec()
-                })
-                .unwrap_or_default(),
+            parallelism: info.1.as_ref().map(|p| p.parallelism).unwrap_or_default() as u32,
             property: info.1.as_ref().map(|p| PbProperty {
                 is_streaming: p.is_streaming,
                 is_serving: p.is_serving,
@@ -102,6 +88,7 @@ impl From<WorkerInfo> for PbWorkerNode {
             transactional_id: info.0.transaction_id.map(|id| id as _),
             resource: info.2.resource,
             started_at: info.2.started_at,
+            node_label: "".to_string(),
         }
     }
 }
@@ -367,8 +354,8 @@ impl ClusterController {
             .await
     }
 
-    pub async fn list_active_parallel_units(&self) -> MetaResult<Vec<ParallelUnit>> {
-        self.inner.read().await.list_active_parallel_units().await
+    pub async fn list_active_worker_slots(&self) -> MetaResult<Vec<WorkerSlotId>> {
+        self.inner.read().await.list_active_worker_slots().await
     }
 
     /// Get the cluster info used for scheduling a streaming job, containing all nodes that are
@@ -392,6 +379,10 @@ impl ClusterController {
             .await
             .get_worker_extra_info_by_id(worker_id)
     }
+
+    pub fn cluster_id(&self) -> &ClusterId {
+        self.env.cluster_id()
+    }
 }
 
 #[derive(Default, Clone)]
@@ -466,7 +457,7 @@ fn meta_node_info(host: &str, started_at: Option<u64>) -> PbWorkerNode {
             .map(HostAddr::to_protobuf)
             .ok(),
         state: PbState::Running as _,
-        parallel_units: vec![],
+        parallelism: 0,
         property: None,
         transactional_id: None,
         resource: Some(risingwave_pb::common::worker_node::Resource {
@@ -475,6 +466,7 @@ fn meta_node_info(host: &str, started_at: Option<u64>) -> PbWorkerNode {
             total_cpu_cores: total_cpu_available() as _,
         }),
         started_at,
+        node_label: "".to_string(),
     }
 }
 
@@ -599,13 +591,6 @@ impl ClusterControllerInner {
     ) -> MetaResult<WorkerId> {
         let txn = self.db.begin().await?;
 
-        // TODO: remove this workaround when we deprecate parallel unit ids.
-        let derive_parallel_units = |txn_id: TransactionId, start: i32, end: i32| {
-            (start..end)
-                .map(|idx| (idx << Self::MAX_WORKER_REUSABLE_ID_BITS) + txn_id)
-                .collect_vec()
-        };
-
         let worker = Worker::find()
             .filter(
                 worker::Column::Host
@@ -620,28 +605,26 @@ impl ClusterControllerInner {
             assert_eq!(worker.worker_type, r#type.into());
             return if worker.worker_type == WorkerType::ComputeNode {
                 let property = property.unwrap();
-                let txn_id = worker.transaction_id.unwrap();
-                let mut current_parallelism = property.parallel_unit_ids.0.clone();
+                let mut current_parallelism = property.parallelism as usize;
                 let new_parallelism = add_property.worker_node_parallelism as usize;
-
-                match new_parallelism.cmp(&current_parallelism.len()) {
+                match new_parallelism.cmp(&current_parallelism) {
                     Ordering::Less => {
                         if !self.disable_automatic_parallelism_control {
                             // Handing over to the subsequent recovery loop for a forced reschedule.
                             tracing::info!(
                                 "worker {} parallelism reduced from {} to {}",
                                 worker.worker_id,
-                                current_parallelism.len(),
+                                current_parallelism,
                                 new_parallelism
                             );
-                            current_parallelism.truncate(new_parallelism);
+                            current_parallelism = new_parallelism;
                         } else {
                             // Warn and keep the original parallelism if the worker registered with a
                             // smaller parallelism.
                             tracing::warn!(
                                 "worker {} parallelism is less than current, current is {}, but received {}",
                                 worker.worker_id,
-                                current_parallelism.len(),
+                                current_parallelism,
                                 new_parallelism
                             );
                         }
@@ -650,14 +633,10 @@ impl ClusterControllerInner {
                         tracing::info!(
                             "worker {} parallelism updated from {} to {}",
                             worker.worker_id,
-                            current_parallelism.len(),
+                            current_parallelism,
                             new_parallelism
                         );
-                        current_parallelism.extend(derive_parallel_units(
-                            txn_id,
-                            current_parallelism.len() as _,
-                            new_parallelism as _,
-                        ));
+                        current_parallelism = new_parallelism;
                     }
                     Ordering::Equal => {}
                 }
@@ -666,7 +645,7 @@ impl ClusterControllerInner {
                 // keep `is_unschedulable` unchanged.
                 property.is_streaming = Set(add_property.is_streaming);
                 property.is_serving = Set(add_property.is_serving);
-                property.parallel_unit_ids = Set(I32Array(current_parallelism));
+                property.parallelism = Set(current_parallelism as _);
 
                 WorkerProperty::update(property).exec(&txn).await?;
                 txn.commit().await?;
@@ -694,11 +673,10 @@ impl ClusterControllerInner {
         if r#type == PbWorkerType::ComputeNode {
             let property = worker_property::ActiveModel {
                 worker_id: Set(worker_id),
-                parallel_unit_ids: Set(I32Array(derive_parallel_units(
-                    *txn_id.as_ref().unwrap(),
-                    0,
-                    add_property.worker_node_parallelism as _,
-                ))),
+                parallelism: Set(add_property
+                    .worker_node_parallelism
+                    .try_into()
+                    .expect("invalid parallelism")),
                 is_streaming: Set(add_property.is_streaming),
                 is_serving: Set(add_property.is_serving),
                 is_unschedulable: Set(add_property.is_unschedulable),
@@ -843,23 +821,20 @@ impl ClusterControllerInner {
             .collect_vec())
     }
 
-    pub async fn list_active_parallel_units(&self) -> MetaResult<Vec<ParallelUnit>> {
-        let parallel_units: Vec<(WorkerId, I32Array)> = WorkerProperty::find()
+    pub async fn list_active_worker_slots(&self) -> MetaResult<Vec<WorkerSlotId>> {
+        let worker_parallelisms: Vec<(WorkerId, i32)> = WorkerProperty::find()
             .select_only()
             .column(worker_property::Column::WorkerId)
-            .column(worker_property::Column::ParallelUnitIds)
+            .column(worker_property::Column::Parallelism)
             .inner_join(Worker)
             .filter(worker::Column::Status.eq(WorkerStatus::Running))
             .into_tuple()
             .all(&self.db)
             .await?;
-        Ok(parallel_units
+        Ok(worker_parallelisms
             .into_iter()
-            .flat_map(|(id, pu)| {
-                pu.0.into_iter().map(move |parallel_unit_id| ParallelUnit {
-                    id: parallel_unit_id as _,
-                    worker_node_id: id as _,
-                })
+            .flat_map(|(worker_id, parallelism)| {
+                (0..parallelism).map(move |idx| WorkerSlotId::new(worker_id as u32, idx as usize))
             })
             .collect_vec())
     }
@@ -889,32 +864,22 @@ impl ClusterControllerInner {
     pub async fn get_streaming_cluster_info(&self) -> MetaResult<StreamingClusterInfo> {
         let mut streaming_workers = self.list_active_streaming_workers().await?;
 
-        let unschedulable_worker_node = streaming_workers
+        let unschedulable_workers = streaming_workers
             .extract_if(|worker| {
                 worker
                     .property
                     .as_ref()
                     .map_or(false, |p| p.is_unschedulable)
             })
-            .collect_vec();
+            .map(|w| w.id)
+            .collect();
 
         let active_workers: HashMap<_, _> =
             streaming_workers.into_iter().map(|w| (w.id, w)).collect();
 
-        let active_parallel_units = active_workers
-            .values()
-            .flat_map(|worker| worker.parallel_units.iter().map(|p| (p.id, p.clone())))
-            .collect();
-
-        let unschedulable_parallel_units = unschedulable_worker_node
-            .iter()
-            .flat_map(|worker| worker.parallel_units.iter().map(|p| (p.id, p.clone())))
-            .collect();
-
         Ok(StreamingClusterInfo {
             worker_nodes: active_workers,
-            parallel_units: active_parallel_units,
-            unschedulable_parallel_units,
+            unschedulable_workers,
         })
     }
 
@@ -977,8 +942,8 @@ mod tests {
             );
         }
 
-        // Since no worker is active, the parallel unit count should be 0.
-        assert_eq!(cluster_ctl.list_active_parallel_units().await?.len(), 0);
+        // Since no worker is active, the parallelism should be 0.
+        assert_eq!(cluster_ctl.list_active_worker_slots().await?.len(), 0);
 
         for id in &worker_ids {
             cluster_ctl.activate_worker(*id).await?;
@@ -997,7 +962,7 @@ mod tests {
             worker_count
         );
         assert_eq!(
-            cluster_ctl.list_active_parallel_units().await?.len(),
+            cluster_ctl.list_active_worker_slots().await?.len(),
             parallelism_num * worker_count
         );
 
@@ -1022,9 +987,9 @@ mod tests {
             cluster_ctl.list_active_serving_workers().await?.len(),
             worker_count - 1
         );
-        let parallel_units = cluster_ctl.list_active_parallel_units().await?;
-        assert!(parallel_units.iter().map(|pu| pu.id).all_unique());
-        assert_eq!(parallel_units.len(), parallelism_num * (worker_count + 1));
+        let worker_slots = cluster_ctl.list_active_worker_slots().await?;
+        assert!(worker_slots.iter().all_unique());
+        assert_eq!(worker_slots.len(), parallelism_num * (worker_count + 1));
 
         // delete workers.
         for host in hosts {
@@ -1032,7 +997,7 @@ mod tests {
         }
         assert_eq!(cluster_ctl.list_active_streaming_workers().await?.len(), 0);
         assert_eq!(cluster_ctl.list_active_serving_workers().await?.len(), 0);
-        assert_eq!(cluster_ctl.list_active_parallel_units().await?.len(), 0);
+        assert_eq!(cluster_ctl.list_active_worker_slots().await?.len(), 0);
 
         Ok(())
     }
diff --git a/src/meta/src/controller/fragment.rs b/src/meta/src/controller/fragment.rs
index e88046ed17467..6c24a6a44e5b4 100644
--- a/src/meta/src/controller/fragment.rs
+++ b/src/meta/src/controller/fragment.rs
@@ -19,16 +19,17 @@ use std::mem::swap;
 use anyhow::Context;
 use itertools::Itertools;
 use risingwave_common::bail;
-use risingwave_common::hash::ParallelUnitMapping;
+use risingwave_common::hash::WorkerSlotId;
 use risingwave_common::util::stream_graph_visitor::visit_stream_node;
 use risingwave_meta_model_v2::actor::ActorStatus;
+use risingwave_meta_model_v2::fragment::DistributionType;
 use risingwave_meta_model_v2::prelude::{Actor, ActorDispatcher, Fragment, Sink, StreamingJob};
 use risingwave_meta_model_v2::{
-    actor, actor_dispatcher, fragment, object, sink, streaming_job, ActorId, ActorUpstreamActors,
-    ConnectorSplits, ExprContext, FragmentId, FragmentVnodeMapping, I32Array, JobStatus, ObjectId,
-    SinkId, SourceId, StreamNode, StreamingParallelism, TableId, VnodeBitmap, WorkerId,
+    actor, actor_dispatcher, fragment, sink, streaming_job, ActorId, ActorUpstreamActors,
+    ConnectorSplits, ExprContext, FragmentId, I32Array, JobStatus, ObjectId, SinkId, SourceId,
+    StreamNode, StreamingParallelism, TableId, VnodeBitmap, WorkerId,
 };
-use risingwave_pb::common::PbParallelUnit;
+use risingwave_pb::common::PbActorLocation;
 use risingwave_pb::meta::subscribe_response::{
     Info as NotificationInfo, Operation as NotificationOperation,
 };
@@ -41,19 +42,19 @@ use risingwave_pb::meta::{
 use risingwave_pb::source::PbConnectorSplits;
 use risingwave_pb::stream_plan::stream_node::NodeBody;
 use risingwave_pb::stream_plan::{
-    PbDispatchStrategy, PbFragmentTypeFlag, PbStreamActor, PbStreamContext,
+    DispatchStrategy, PbDispatchStrategy, PbFragmentTypeFlag, PbStreamActor, PbStreamContext,
 };
-use sea_orm::sea_query::{Expr, Value};
+use sea_orm::sea_query::Expr;
 use sea_orm::ActiveValue::Set;
 use sea_orm::{
-    ActiveModelTrait, ColumnTrait, EntityTrait, JoinType, ModelTrait, PaginatorTrait, QueryFilter,
-    QuerySelect, RelationTrait, TransactionTrait,
+    ColumnTrait, EntityTrait, JoinType, ModelTrait, PaginatorTrait, QueryFilter, QuerySelect,
+    RelationTrait, TransactionTrait, Value,
 };
 
 use crate::controller::catalog::{CatalogController, CatalogControllerInner};
 use crate::controller::utils::{
-    get_actor_dispatchers, get_parallel_unit_to_worker_map, FragmentDesc, PartialActorLocation,
-    PartialFragmentStateTables,
+    get_actor_dispatchers, get_fragment_mappings, rebuild_fragment_mapping_from_actors,
+    FragmentDesc, PartialActorLocation, PartialFragmentStateTables,
 };
 use crate::manager::{ActorInfos, InflightFragmentInfo, LocalNotification};
 use crate::model::{TableFragments, TableParallelism};
@@ -67,24 +68,22 @@ impl CatalogControllerInner {
     ) -> MetaResult<impl Iterator<Item = FragmentWorkerSlotMapping> + '_> {
         let txn = self.db.begin().await?;
 
-        let fragment_mappings: Vec<(FragmentId, FragmentVnodeMapping)> = Fragment::find()
-            .join(JoinType::InnerJoin, fragment::Relation::Object.def())
-            .join(JoinType::InnerJoin, object::Relation::StreamingJob.def())
+        let job_ids: Vec<ObjectId> = StreamingJob::find()
             .select_only()
-            .columns([fragment::Column::FragmentId, fragment::Column::VnodeMapping])
+            .column(streaming_job::Column::JobId)
             .filter(streaming_job::Column::JobStatus.eq(JobStatus::Created))
             .into_tuple()
             .all(&txn)
             .await?;
 
-        let parallel_unit_to_worker = get_parallel_unit_to_worker_map(&txn).await?;
+        let mut result = vec![];
+        for job_id in job_ids {
+            let mappings = get_fragment_mappings(&txn, job_id).await?;
 
-        let mappings = CatalogController::convert_fragment_mappings(
-            fragment_mappings,
-            &parallel_unit_to_worker,
-        )?;
+            result.extend(mappings.into_iter());
+        }
 
-        Ok(mappings.into_iter())
+        Ok(result.into_iter())
     }
 }
 
@@ -183,9 +182,9 @@ impl CatalogController {
             fragment_type_mask: pb_fragment_type_mask,
             distribution_type: pb_distribution_type,
             actors: pb_actors,
-            vnode_mapping: pb_vnode_mapping,
             state_table_ids: pb_state_table_ids,
             upstream_fragment_ids: pb_upstream_fragment_ids,
+            ..
         } = pb_fragment;
 
         let state_table_ids = pb_state_table_ids.into();
@@ -210,7 +209,7 @@ impl CatalogController {
         for mut actor in pb_actors {
             let mut upstream_actors = BTreeMap::new();
 
-            let node = actor.nodes.as_mut().context("nodes is empty")?;
+            let node = actor.nodes.as_mut().context("nodes are empty")?;
 
             visit_stream_node(node, |body| {
                 if let NodeBody::Merge(m) = body {
@@ -245,11 +244,7 @@ impl CatalogController {
                 )
             })?;
 
-            let (worker_id, parallel_unit_id) = status
-                .parallel_unit
-                .as_ref()
-                .map(|pu| (pu.worker_node_id as WorkerId, pu.id as i32))
-                .expect("no parallel unit id found in actor_status");
+            let worker_id = status.worker_id() as _;
 
             assert_eq!(
                 pb_upstream_actor_id
@@ -269,7 +264,6 @@ impl CatalogController {
                 fragment_id: fragment_id as _,
                 status: status.get_state().unwrap().into(),
                 splits,
-                parallel_unit_id,
                 worker_id,
                 upstream_actor_ids: upstream_actors.into(),
                 vnode_bitmap: pb_vnode_bitmap.as_ref().map(VnodeBitmap::from),
@@ -286,11 +280,6 @@ impl CatalogController {
 
         let upstream_fragment_id = pb_upstream_fragment_ids.into();
 
-        let vnode_mapping = pb_vnode_mapping
-            .as_ref()
-            .map(FragmentVnodeMapping::from)
-            .unwrap();
-
         let stream_node = StreamNode::from(&stream_node);
 
         let distribution_type = PbFragmentDistributionType::try_from(pb_distribution_type)
@@ -303,7 +292,6 @@ impl CatalogController {
             fragment_type_mask: pb_fragment_type_mask as _,
             distribution_type,
             stream_node,
-            vnode_mapping,
             state_table_ids,
             upstream_fragment_id,
         };
@@ -352,6 +340,8 @@ impl CatalogController {
                 }
                 .into(),
             ),
+            node_label: "".to_string(),
+            backfill_done: true,
         };
 
         Ok(table_fragments)
@@ -373,7 +363,6 @@ impl CatalogController {
             fragment_type_mask,
             distribution_type,
             stream_node,
-            vnode_mapping,
             state_table_ids,
             upstream_fragment_id,
         } = fragment;
@@ -399,7 +388,6 @@ impl CatalogController {
                 actor_id,
                 fragment_id,
                 status,
-                parallel_unit_id,
                 worker_id,
                 splits,
                 upstream_actor_ids,
@@ -444,10 +432,7 @@ impl CatalogController {
             pb_actor_status.insert(
                 actor_id as _,
                 PbActorStatus {
-                    parallel_unit: Some(PbParallelUnit {
-                        id: parallel_unit_id as _,
-                        worker_node_id: worker_id as _,
-                    }),
+                    location: PbActorLocation::from_worker(worker_id as u32),
                     state: PbActorState::from(status) as _,
                 },
             );
@@ -469,7 +454,6 @@ impl CatalogController {
         }
 
         let pb_upstream_fragment_ids = upstream_fragment_id.into_u32_array();
-        let pb_vnode_mapping = vnode_mapping.to_protobuf();
         let pb_state_table_ids = state_table_ids.into_u32_array();
         let pb_distribution_type = PbFragmentDistributionType::from(distribution_type) as _;
         let pb_fragment = PbFragment {
@@ -477,7 +461,6 @@ impl CatalogController {
             fragment_type_mask: fragment_type_mask as _,
             distribution_type: pb_distribution_type,
             actors: pb_actors,
-            vnode_mapping: Some(pb_vnode_mapping),
             state_table_ids: pb_state_table_ids,
             upstream_fragment_ids: pb_upstream_fragment_ids,
         };
@@ -907,93 +890,133 @@ impl CatalogController {
         Ok(ActorInfos::new(fragment_infos))
     }
 
-    pub async fn migrate_actors(&self, plan: HashMap<i32, PbParallelUnit>) -> MetaResult<()> {
+    pub async fn migrate_actors(
+        &self,
+        plan: HashMap<WorkerSlotId, WorkerSlotId>,
+    ) -> MetaResult<()> {
         let inner = self.inner.read().await;
         let txn = inner.db.begin().await?;
-        for (from_pu_id, to_pu_id) in &plan {
-            Actor::update_many()
-                .col_expr(
-                    actor::Column::ParallelUnitId,
-                    Expr::value(Value::Int(Some(to_pu_id.id as i32))),
-                )
-                .col_expr(
-                    actor::Column::WorkerId,
-                    Expr::value(Value::Int(Some(to_pu_id.worker_node_id as WorkerId))),
-                )
-                .filter(actor::Column::ParallelUnitId.eq(*from_pu_id))
-                .exec(&txn)
-                .await?;
-        }
 
-        let fragment_mapping: Vec<(FragmentId, FragmentVnodeMapping)> = Fragment::find()
+        let actors: Vec<(
+            FragmentId,
+            DistributionType,
+            ActorId,
+            Option<VnodeBitmap>,
+            WorkerId,
+            ActorStatus,
+        )> = Actor::find()
             .select_only()
-            .columns([fragment::Column::FragmentId, fragment::Column::VnodeMapping])
-            .join(JoinType::InnerJoin, fragment::Relation::Actor.def())
-            .filter(actor::Column::ParallelUnitId.is_in(plan.keys().cloned().collect::<Vec<_>>()))
+            .columns([
+                fragment::Column::FragmentId,
+                fragment::Column::DistributionType,
+            ])
+            .columns([
+                actor::Column::ActorId,
+                actor::Column::VnodeBitmap,
+                actor::Column::WorkerId,
+                actor::Column::Status,
+            ])
+            .join(JoinType::InnerJoin, actor::Relation::Fragment.def())
             .into_tuple()
             .all(&txn)
             .await?;
-        // TODO: we'd better not store vnode mapping in fragment table and derive it from actors.
 
-        for (fragment_id, vnode_mapping) in &fragment_mapping {
-            let mut pb_vnode_mapping = vnode_mapping.to_protobuf();
-            pb_vnode_mapping.data.iter_mut().for_each(|id| {
-                if let Some(new_id) = plan.get(&(*id as i32)) {
-                    *id = new_id.id;
-                }
-            });
-            fragment::ActiveModel {
-                fragment_id: Set(*fragment_id),
-                vnode_mapping: Set(FragmentVnodeMapping::from(&pb_vnode_mapping)),
-                ..Default::default()
+        let mut actor_locations = HashMap::new();
+
+        for (fragment_id, _, actor_id, _, worker_id, status) in &actors {
+            if *status != ActorStatus::Running {
+                tracing::warn!(
+                    "skipping actor {} in fragment {} with status {:?}",
+                    actor_id,
+                    fragment_id,
+                    status
+                );
+                continue;
             }
-            .update(&txn)
-            .await?;
+
+            actor_locations
+                .entry(*worker_id)
+                .or_insert(HashMap::new())
+                .entry(*fragment_id)
+                .or_insert(BTreeSet::new())
+                .insert(*actor_id);
         }
 
-        let parallel_unit_to_worker = get_parallel_unit_to_worker_map(&txn).await?;
+        let expired_workers: HashSet<_> = plan.keys().map(|k| k.worker_id() as WorkerId).collect();
+
+        let mut actor_migration_plan = HashMap::new();
+        for (worker, fragment) in actor_locations {
+            if expired_workers.contains(&worker) {
+                for (_, actors) in fragment {
+                    let worker_slot_to_actor: HashMap<_, _> = actors
+                        .iter()
+                        .enumerate()
+                        .map(|(idx, actor_id)| {
+                            (WorkerSlotId::new(worker as _, idx as _), *actor_id)
+                        })
+                        .collect();
 
-        let fragment_worker_slot_mapping =
-            Self::convert_fragment_mappings(fragment_mapping, &parallel_unit_to_worker)?;
+                    for (worker_slot, actor) in worker_slot_to_actor {
+                        actor_migration_plan
+                            .insert(actor, plan[&worker_slot].worker_id() as WorkerId);
+                    }
+                }
+            }
+        }
+
+        for (actor, worker) in actor_migration_plan {
+            Actor::update_many()
+                .col_expr(
+                    actor::Column::WorkerId,
+                    Expr::value(Value::Int(Some(worker))),
+                )
+                .filter(actor::Column::ActorId.eq(actor))
+                .exec(&txn)
+                .await?;
+        }
 
         txn.commit().await?;
 
-        self.notify_fragment_mapping(NotificationOperation::Update, fragment_worker_slot_mapping)
-            .await;
+        self.notify_fragment_mapping(
+            NotificationOperation::Update,
+            rebuild_fragment_mapping_from_actors(actors),
+        )
+        .await;
 
         Ok(())
     }
 
-    pub(crate) fn convert_fragment_mappings(
-        fragment_mappings: Vec<(FragmentId, FragmentVnodeMapping)>,
-        parallel_unit_to_worker: &HashMap<u32, u32>,
-    ) -> MetaResult<Vec<PbFragmentWorkerSlotMapping>> {
-        let mut result = vec![];
-
-        for (fragment_id, mapping) in fragment_mappings {
-            result.push(PbFragmentWorkerSlotMapping {
-                fragment_id: fragment_id as _,
-                mapping: Some(
-                    ParallelUnitMapping::from_protobuf(&mapping.to_protobuf())
-                        .to_worker_slot(parallel_unit_to_worker)?
-                        .to_protobuf(),
-                ),
-            })
-        }
-
-        Ok(result)
-    }
-
-    pub async fn all_inuse_parallel_units(&self) -> MetaResult<Vec<i32>> {
+    pub async fn all_inuse_worker_slots(&self) -> MetaResult<HashSet<WorkerSlotId>> {
         let inner = self.inner.read().await;
-        let parallel_units: Vec<i32> = Actor::find()
+
+        let actors: Vec<(FragmentId, ActorId, WorkerId)> = Actor::find()
             .select_only()
-            .column(actor::Column::ParallelUnitId)
-            .distinct()
+            .columns([fragment::Column::FragmentId])
+            .columns([actor::Column::ActorId, actor::Column::WorkerId])
+            .join(JoinType::InnerJoin, actor::Relation::Fragment.def())
             .into_tuple()
             .all(&inner.db)
             .await?;
-        Ok(parallel_units)
+
+        let mut actor_locations = HashMap::new();
+
+        for (fragment_id, _, worker_id) in actors {
+            *actor_locations
+                .entry(worker_id)
+                .or_insert(HashMap::new())
+                .entry(fragment_id)
+                .or_insert(0_usize) += 1;
+        }
+
+        let mut result = HashSet::new();
+        for (worker_id, mapping) in actor_locations {
+            let max_fragment_len = mapping.values().max().unwrap();
+
+            result
+                .extend((0..*max_fragment_len).map(|idx| WorkerSlotId::new(worker_id as u32, idx)))
+        }
+
+        Ok(result)
     }
 
     pub async fn all_node_actors(
@@ -1035,10 +1058,7 @@ impl CatalogController {
             let (table_fragments, actor_status, _) =
                 Self::compose_fragment(fragment, actors, dispatcher_info)?;
             for actor in table_fragments.actors {
-                let node_id = actor_status[&actor.actor_id]
-                    .get_parallel_unit()
-                    .unwrap()
-                    .worker_node_id as WorkerId;
+                let node_id = actor_status[&actor.actor_id].worker_id() as WorkerId;
                 node_actors
                     .entry(node_id)
                     .or_insert_with(Vec::new)
@@ -1159,7 +1179,7 @@ impl CatalogController {
     pub async fn get_upstream_root_fragments(
         &self,
         upstream_job_ids: Vec<ObjectId>,
-    ) -> MetaResult<HashMap<ObjectId, PbFragment>> {
+    ) -> MetaResult<(HashMap<ObjectId, PbFragment>, Vec<(ActorId, WorkerId)>)> {
         let inner = self.inner.read().await;
 
         let all_upstream_fragments = Fragment::find()
@@ -1192,14 +1212,24 @@ impl CatalogController {
             );
         }
 
-        Ok(root_fragments)
+        let actors: Vec<(ActorId, WorkerId)> = Actor::find()
+            .select_only()
+            .columns([actor::Column::ActorId, actor::Column::WorkerId])
+            .into_tuple()
+            .all(&inner.db)
+            .await?;
+
+        Ok((root_fragments, actors))
     }
 
     /// Get the downstream `Chain` fragments of the specified table.
     pub async fn get_downstream_chain_fragments(
         &self,
         job_id: ObjectId,
-    ) -> MetaResult<Vec<(PbDispatchStrategy, PbFragment)>> {
+    ) -> MetaResult<(
+        Vec<(DispatchStrategy, PbFragment)>,
+        Vec<(ActorId, WorkerId)>,
+    )> {
         let mview_fragment = self.get_mview_fragment(job_id).await?;
         let downstream_dispatches: HashMap<_, _> = mview_fragment.actors[0]
             .dispatcher
@@ -1236,7 +1266,14 @@ impl CatalogController {
             chain_fragments.push((dispatch_strategy, fragment));
         }
 
-        Ok(chain_fragments)
+        let actors: Vec<(ActorId, WorkerId)> = Actor::find()
+            .select_only()
+            .columns([actor::Column::ActorId, actor::Column::WorkerId])
+            .into_tuple()
+            .all(&inner.db)
+            .await?;
+
+        Ok((chain_fragments, actors))
     }
 
     pub async fn load_source_fragment_ids(
@@ -1379,17 +1416,16 @@ mod tests {
     use std::collections::{BTreeMap, HashMap};
 
     use itertools::Itertools;
-    use risingwave_common::hash::{ParallelUnitId, ParallelUnitMapping};
+    use risingwave_common::hash::ActorMapping;
     use risingwave_common::util::iter_util::ZipEqDebug;
     use risingwave_common::util::stream_graph_visitor::visit_stream_node;
     use risingwave_meta_model_v2::actor::ActorStatus;
     use risingwave_meta_model_v2::fragment::DistributionType;
     use risingwave_meta_model_v2::{
         actor, actor_dispatcher, fragment, ActorId, ActorUpstreamActors, ConnectorSplits,
-        ExprContext, FragmentId, FragmentVnodeMapping, I32Array, ObjectId, StreamNode, TableId,
-        VnodeBitmap,
+        ExprContext, FragmentId, I32Array, ObjectId, StreamNode, TableId, VnodeBitmap,
     };
-    use risingwave_pb::common::ParallelUnit;
+    use risingwave_pb::common::PbActorLocation;
     use risingwave_pb::meta::table_fragments::actor_status::PbActorState;
     use risingwave_pb::meta::table_fragments::fragment::PbFragmentDistributionType;
     use risingwave_pb::meta::table_fragments::{PbActorStatus, PbFragment};
@@ -1412,15 +1448,6 @@ mod tests {
 
     const TEST_STATE_TABLE_ID: TableId = 1000;
 
-    fn generate_parallel_units(count: u32) -> Vec<ParallelUnit> {
-        (0..count)
-            .map(|parallel_unit_id| ParallelUnit {
-                id: parallel_unit_id,
-                worker_node_id: 0,
-            })
-            .collect_vec()
-    }
-
     fn generate_dispatchers_for_actor(actor_id: u32) -> Vec<Dispatcher> {
         vec![PbDispatcher {
             r#type: PbDispatcherType::Hash as _,
@@ -1465,10 +1492,6 @@ mod tests {
     #[tokio::test]
     async fn test_extract_fragment() -> MetaResult<()> {
         let actor_count = 3u32;
-        let parallel_units = generate_parallel_units(actor_count);
-        let parallel_unit_mapping = ParallelUnitMapping::build(&parallel_units);
-        let actor_vnode_bitmaps = parallel_unit_mapping.to_bitmaps();
-
         let upstream_actor_ids: HashMap<ActorId, BTreeMap<FragmentId, Vec<ActorId>>> = (0
             ..actor_count)
             .map(|actor_id| {
@@ -1479,6 +1502,9 @@ mod tests {
             })
             .collect();
 
+        let actor_bitmaps =
+            ActorMapping::new_uniform((0..actor_count).map(|i| i as _)).to_bitmaps();
+
         let pb_actors = (0..actor_count)
             .map(|actor_id| {
                 let actor_upstream_actor_ids =
@@ -1495,7 +1521,7 @@ mod tests {
                         .flatten()
                         .map(|id| *id as _)
                         .collect(),
-                    vnode_bitmap: actor_vnode_bitmaps
+                    vnode_bitmap: actor_bitmaps
                         .get(&actor_id)
                         .cloned()
                         .map(|bitmap| bitmap.to_protobuf()),
@@ -1512,7 +1538,6 @@ mod tests {
             fragment_type_mask: PbFragmentTypeFlag::Source as _,
             distribution_type: PbFragmentDistributionType::Hash as _,
             actors: pb_actors.clone(),
-            vnode_mapping: Some(parallel_unit_mapping.to_protobuf()),
             state_table_ids: vec![TEST_STATE_TABLE_ID as _],
             upstream_fragment_ids: upstream_actor_ids
                 .values()
@@ -1525,7 +1550,7 @@ mod tests {
                 (
                     actor_id,
                     PbActorStatus {
-                        parallel_unit: Some(parallel_units[actor_id as usize].clone()),
+                        location: PbActorLocation::from_worker(0),
                         state: PbActorState::Running as _,
                     },
                 )
@@ -1579,9 +1604,6 @@ mod tests {
     #[tokio::test]
     async fn test_compose_fragment() -> MetaResult<()> {
         let actor_count = 3u32;
-        let parallel_units = generate_parallel_units(actor_count);
-        let parallel_unit_mapping = ParallelUnitMapping::build(&parallel_units);
-        let mut actor_vnode_bitmaps = parallel_unit_mapping.to_bitmaps();
 
         let upstream_actor_ids: HashMap<ActorId, BTreeMap<FragmentId, Vec<ActorId>>> = (0
             ..actor_count)
@@ -1593,14 +1615,11 @@ mod tests {
             })
             .collect();
 
+        let mut actor_bitmaps =
+            ActorMapping::new_uniform((0..actor_count).map(|i| i as _)).to_bitmaps();
+
         let actors = (0..actor_count)
             .map(|actor_id| {
-                let parallel_unit_id = actor_id as ParallelUnitId;
-
-                let vnode_bitmap = actor_vnode_bitmaps
-                    .remove(&parallel_unit_id)
-                    .map(|m| VnodeBitmap::from(&m.to_protobuf()));
-
                 let actor_splits = Some(ConnectorSplits::from(&PbConnectorSplits {
                     splits: vec![PbConnectorSplit {
                         split_type: "dummy".to_string(),
@@ -1616,10 +1635,13 @@ mod tests {
                     fragment_id: TEST_FRAGMENT_ID,
                     status: ActorStatus::Running,
                     splits: actor_splits,
-                    parallel_unit_id: parallel_unit_id as i32,
                     worker_id: 0,
                     upstream_actor_ids: ActorUpstreamActors(actor_upstream_actor_ids),
-                    vnode_bitmap,
+                    vnode_bitmap: actor_bitmaps
+                        .remove(&actor_id)
+                        .map(|bitmap| bitmap.to_protobuf())
+                        .as_ref()
+                        .map(VnodeBitmap::from),
                     expr_context: ExprContext::from(&PbExprContext {
                         time_zone: String::from("America/New_York"),
                     }),
@@ -1659,7 +1681,6 @@ mod tests {
             fragment_type_mask: 0,
             distribution_type: DistributionType::Hash,
             stream_node: StreamNode::from(&stream_node),
-            vnode_mapping: FragmentVnodeMapping::from(&parallel_unit_mapping.to_protobuf()),
             state_table_ids: I32Array(vec![TEST_STATE_TABLE_ID]),
             upstream_fragment_id: I32Array::default(),
         };
@@ -1674,14 +1695,6 @@ mod tests {
         assert_eq!(pb_actor_status.len(), actor_count as usize);
         assert_eq!(pb_actor_splits.len(), actor_count as usize);
 
-        for (actor_id, actor_status) in &pb_actor_status {
-            let parallel_unit_id = parallel_units[*actor_id as usize].id;
-            assert_eq!(
-                parallel_unit_id,
-                actor_status.parallel_unit.clone().unwrap().id
-            );
-        }
-
         let pb_actors = pb_fragment.actors.clone();
 
         check_fragment_template(fragment.clone(), pb_actors.clone(), &upstream_actor_ids);
@@ -1703,7 +1716,6 @@ mod tests {
                 fragment_id,
                 status,
                 splits,
-                parallel_unit_id,
                 worker_id: _,
                 upstream_actor_ids,
                 vnode_bitmap,
@@ -1723,7 +1735,6 @@ mod tests {
         {
             assert_eq!(actor_id, pb_actor_id as ActorId);
             assert_eq!(fragment_id, pb_fragment_id as FragmentId);
-            assert_eq!(parallel_unit_id, pb_actor_id as i32);
             let upstream_actor_ids = upstream_actor_ids.into_inner();
 
             assert_eq!(
@@ -1779,7 +1790,6 @@ mod tests {
             fragment_type_mask,
             distribution_type: pb_distribution_type,
             actors: _,
-            vnode_mapping: pb_vnode_mapping,
             state_table_ids: pb_state_table_ids,
             upstream_fragment_ids: pb_upstream_fragment_ids,
         } = pb_fragment;
@@ -1790,10 +1800,6 @@ mod tests {
             pb_distribution_type,
             PbFragmentDistributionType::from(fragment.distribution_type) as i32
         );
-        assert_eq!(
-            pb_vnode_mapping.unwrap(),
-            fragment.vnode_mapping.to_protobuf()
-        );
 
         assert_eq!(
             pb_upstream_fragment_ids,
diff --git a/src/meta/src/controller/mod.rs b/src/meta/src/controller/mod.rs
index 5ace222739d52..43078ea812723 100644
--- a/src/meta/src/controller/mod.rs
+++ b/src/meta/src/controller/mod.rs
@@ -200,6 +200,7 @@ impl From<ObjectModel<source::Model>> for PbSource {
             initialized_at_cluster_version: value.1.initialized_at_cluster_version,
             created_at_cluster_version: value.1.created_at_cluster_version,
             secret_refs: secret_ref_map,
+            rate_limit: value.0.rate_limit.map(|v| v as _),
         }
     }
 }
@@ -240,6 +241,7 @@ impl From<ObjectModel<sink::Model>> for PbSink {
             created_at_cluster_version: value.1.created_at_cluster_version,
             create_type: PbCreateType::Foreground as _,
             secret_refs: secret_ref_map,
+            original_target_columns: value.0.original_target_columns.to_protobuf(),
         }
     }
 }
diff --git a/src/meta/src/controller/streaming_job.rs b/src/meta/src/controller/streaming_job.rs
index 42cf44f184cf7..f5d97eca1c564 100644
--- a/src/meta/src/controller/streaming_job.rs
+++ b/src/meta/src/controller/streaming_job.rs
@@ -16,8 +16,6 @@ use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
 use std::num::NonZeroUsize;
 
 use itertools::Itertools;
-use risingwave_common::buffer::Bitmap;
-use risingwave_common::hash::{ActorMapping, ParallelUnitId, ParallelUnitMapping};
 use risingwave_common::util::column_index_mapping::ColIndexMapping;
 use risingwave_common::util::stream_graph_visitor::visit_stream_node;
 use risingwave_common::{bail, current_cluster_version};
@@ -30,26 +28,25 @@ use risingwave_meta_model_v2::prelude::{
 };
 use risingwave_meta_model_v2::{
     actor, actor_dispatcher, fragment, index, object, object_dependency, sink, source,
-    streaming_job, table, ActorId, ActorUpstreamActors, CreateType, DatabaseId, ExprNodeArray,
-    FragmentId, I32Array, IndexId, JobStatus, ObjectId, SchemaId, SourceId, StreamNode,
-    StreamingParallelism, TableId, TableVersion, UserId,
+    streaming_job, table, ActorId, ActorUpstreamActors, ColumnCatalogArray, CreateType, DatabaseId,
+    ExprNodeArray, FragmentId, I32Array, IndexId, JobStatus, ObjectId, SchemaId, SourceId,
+    StreamNode, StreamingParallelism, TableId, TableVersion, UserId,
 };
 use risingwave_pb::catalog::source::PbOptionalAssociatedTableId;
 use risingwave_pb::catalog::table::{PbOptionalAssociatedSourceId, PbTableVersion};
 use risingwave_pb::catalog::{PbCreateType, PbTable};
-use risingwave_pb::meta::relation::PbRelationInfo;
+use risingwave_pb::meta::relation::{PbRelationInfo, RelationInfo};
 use risingwave_pb::meta::subscribe_response::{
-    Info as NotificationInfo, Operation as NotificationOperation, Operation,
+    Info as NotificationInfo, Info, Operation as NotificationOperation, Operation,
 };
-use risingwave_pb::meta::table_fragments::PbActorStatus;
 use risingwave_pb::meta::{
-    FragmentWorkerSlotMapping, PbFragmentWorkerSlotMapping, PbRelation, PbRelationGroup,
-    PbTableFragments, Relation,
+    PbFragmentWorkerSlotMapping, PbRelation, PbRelationGroup, PbTableFragments, Relation,
+    RelationGroup,
 };
 use risingwave_pb::source::{PbConnectorSplit, PbConnectorSplits};
 use risingwave_pb::stream_plan::stream_fragment_graph::Parallelism;
 use risingwave_pb::stream_plan::stream_node::PbNodeBody;
-use risingwave_pb::stream_plan::update_mutation::{MergeUpdate, PbMergeUpdate};
+use risingwave_pb::stream_plan::update_mutation::PbMergeUpdate;
 use risingwave_pb::stream_plan::{
     PbDispatcher, PbDispatcherType, PbFragmentTypeFlag, PbStreamActor,
 };
@@ -61,12 +58,13 @@ use sea_orm::{
     TransactionTrait,
 };
 
-use crate::barrier::Reschedule;
+use crate::barrier::{ReplaceTablePlan, Reschedule};
 use crate::controller::catalog::CatalogController;
 use crate::controller::rename::ReplaceTableExprRewriter;
 use crate::controller::utils::{
-    check_relation_name_duplicate, check_sink_into_table_cycle, ensure_object_id, ensure_user_id,
-    get_fragment_actor_ids, get_fragment_mappings, get_parallel_unit_to_worker_map,
+    build_relation_group, check_relation_name_duplicate, check_sink_into_table_cycle,
+    ensure_object_id, ensure_user_id, get_fragment_actor_ids, get_fragment_mappings,
+    rebuild_fragment_mapping_from_actors, PartialObject,
 };
 use crate::controller::ObjectModel;
 use crate::manager::{NotificationVersion, SinkId, StreamingJob};
@@ -157,6 +155,8 @@ impl CatalogController {
             }
         }
 
+        let mut relations = vec![];
+
         match streaming_job {
             StreamingJob::MaterializedView(table) => {
                 let job_id = Self::create_streaming_job_obj(
@@ -171,8 +171,12 @@ impl CatalogController {
                 )
                 .await?;
                 table.id = job_id as _;
-                let table: table::ActiveModel = table.clone().into();
-                Table::insert(table).exec(&txn).await?;
+                let table_model: table::ActiveModel = table.clone().into();
+                Table::insert(table_model).exec(&txn).await?;
+
+                relations.push(Relation {
+                    relation_info: Some(RelationInfo::Table(table.to_owned())),
+                });
             }
             StreamingJob::Sink(sink, _) => {
                 if let Some(target_table_id) = sink.target_table {
@@ -202,8 +206,11 @@ impl CatalogController {
                 )
                 .await?;
                 sink.id = job_id as _;
-                let sink: sink::ActiveModel = sink.clone().into();
-                Sink::insert(sink).exec(&txn).await?;
+                let sink_model: sink::ActiveModel = sink.clone().into();
+                Sink::insert(sink_model).exec(&txn).await?;
+                relations.push(Relation {
+                    relation_info: Some(RelationInfo::Sink(sink.to_owned())),
+                });
             }
             StreamingJob::Table(src, table, _) => {
                 let job_id = Self::create_streaming_job_obj(
@@ -235,9 +242,15 @@ impl CatalogController {
                     );
                     let source: source::ActiveModel = src.clone().into();
                     Source::insert(source).exec(&txn).await?;
+                    relations.push(Relation {
+                        relation_info: Some(RelationInfo::Source(src.to_owned())),
+                    });
                 }
-                let table: table::ActiveModel = table.clone().into();
-                Table::insert(table).exec(&txn).await?;
+                let table_model: table::ActiveModel = table.clone().into();
+                Table::insert(table_model).exec(&txn).await?;
+                relations.push(Relation {
+                    relation_info: Some(RelationInfo::Table(table.to_owned())),
+                });
             }
             StreamingJob::Index(index, table) => {
                 ensure_object_id(ObjectType::Table, index.primary_table_id as _, &txn).await?;
@@ -265,10 +278,16 @@ impl CatalogController {
                 .exec(&txn)
                 .await?;
 
-                let table: table::ActiveModel = table.clone().into();
-                Table::insert(table).exec(&txn).await?;
-                let index: index::ActiveModel = index.clone().into();
-                Index::insert(index).exec(&txn).await?;
+                let table_model: table::ActiveModel = table.clone().into();
+                Table::insert(table_model).exec(&txn).await?;
+                let index_model: index::ActiveModel = index.clone().into();
+                Index::insert(index_model).exec(&txn).await?;
+                relations.push(Relation {
+                    relation_info: Some(RelationInfo::Table(table.to_owned())),
+                });
+                relations.push(Relation {
+                    relation_info: Some(RelationInfo::Index(index.to_owned())),
+                });
             }
             StreamingJob::Source(src) => {
                 let job_id = Self::create_streaming_job_obj(
@@ -283,19 +302,22 @@ impl CatalogController {
                 )
                 .await?;
                 src.id = job_id as _;
-                let source: source::ActiveModel = src.clone().into();
-                Source::insert(source).exec(&txn).await?;
+                let source_model: source::ActiveModel = src.clone().into();
+                Source::insert(source_model).exec(&txn).await?;
+                relations.push(Relation {
+                    relation_info: Some(RelationInfo::Source(src.to_owned())),
+                });
             }
         }
 
-        // get dependent secret ref.
-        let dependent_secret_refs = streaming_job.dependent_secret_refs()?;
+        // get dependent secrets.
+        let dependent_secret_ids = streaming_job.dependent_secret_ids()?;
 
         let dependent_objs = dependent_relations
             .iter()
-            .chain(dependent_secret_refs.iter());
+            .chain(dependent_secret_ids.iter());
         // record object dependency.
-        if !dependent_secret_refs.is_empty() || !dependent_relations.is_empty() {
+        if !dependent_secret_ids.is_empty() || !dependent_relations.is_empty() {
             ObjectDependency::insert_many(dependent_objs.map(|id| {
                 object_dependency::ActiveModel {
                     oid: Set(*id as _),
@@ -309,18 +331,25 @@ impl CatalogController {
 
         txn.commit().await?;
 
+        let _version = self
+            .notify_frontend(
+                Operation::Add,
+                Info::RelationGroup(RelationGroup { relations }),
+            )
+            .await;
+
         Ok(())
     }
 
     pub async fn create_internal_table_catalog(
         &self,
         job_id: ObjectId,
-        internal_tables: Vec<PbTable>,
+        mut internal_tables: Vec<PbTable>,
     ) -> MetaResult<HashMap<u32, u32>> {
         let inner = self.inner.write().await;
         let txn = inner.db.begin().await?;
         let mut table_id_map = HashMap::new();
-        for table in internal_tables {
+        for table in &mut internal_tables {
             let table_id = Self::create_object(
                 &txn,
                 ObjectType::Table,
@@ -331,14 +360,27 @@ impl CatalogController {
             .await?
             .oid;
             table_id_map.insert(table.id, table_id as u32);
-            let mut table: table::ActiveModel = table.into();
-            table.table_id = Set(table_id as _);
-            table.belongs_to_job_id = Set(Some(job_id as _));
-            table.fragment_id = NotSet;
-            Table::insert(table).exec(&txn).await?;
+            table.id = table_id as _;
+            let mut table_model: table::ActiveModel = table.clone().into();
+            table_model.table_id = Set(table_id as _);
+            table_model.belongs_to_job_id = Set(Some(job_id as _));
+            table_model.fragment_id = NotSet;
+            Table::insert(table_model).exec(&txn).await?;
         }
         txn.commit().await?;
-
+        let _version = self
+            .notify_frontend(
+                Operation::Add,
+                Info::RelationGroup(RelationGroup {
+                    relations: internal_tables
+                        .iter()
+                        .map(|table| Relation {
+                            relation_info: Some(RelationInfo::Table(table.clone())),
+                        })
+                        .collect(),
+                }),
+            )
+            .await;
         Ok(table_id_map)
     }
 
@@ -418,7 +460,7 @@ impl CatalogController {
         job_id: ObjectId,
         is_cancelled: bool,
     ) -> MetaResult<bool> {
-        let inner = self.inner.write().await;
+        let mut inner = self.inner.write().await;
         let txn = inner.db.begin().await?;
 
         let cnt = Object::find_by_id(job_id).count(&txn).await?;
@@ -463,6 +505,48 @@ impl CatalogController {
             .one(&txn)
             .await?;
 
+        // Get notification info
+        let mut objs = vec![];
+        let obj: Option<PartialObject> = Object::find_by_id(job_id)
+            .select_only()
+            .columns([
+                object::Column::Oid,
+                object::Column::ObjType,
+                object::Column::SchemaId,
+                object::Column::DatabaseId,
+            ])
+            .into_partial_model()
+            .one(&txn)
+            .await?;
+        let obj = obj.ok_or_else(|| MetaError::catalog_id_not_found("streaming job", job_id))?;
+        objs.push(obj);
+        let internal_table_objs: Vec<PartialObject> = Object::find()
+            .select_only()
+            .columns([
+                object::Column::Oid,
+                object::Column::ObjType,
+                object::Column::SchemaId,
+                object::Column::DatabaseId,
+            ])
+            .join(JoinType::InnerJoin, object::Relation::Table.def())
+            .filter(table::Column::BelongsToJobId.eq(job_id))
+            .into_partial_model()
+            .all(&txn)
+            .await?;
+        objs.extend(internal_table_objs);
+        if let Some(source_id) = associated_source_id {
+            let source_obj = Object::find_by_id(source_id)
+                .select_only()
+                .column(object::Column::ObjType)
+                .into_partial_model()
+                .one(&txn)
+                .await?
+                .ok_or_else(|| MetaError::catalog_id_not_found("source", source_id))?;
+            objs.push(source_obj);
+        }
+        let relation_group = build_relation_group(objs);
+
+        // Can delete objects after queried notification info
         Object::delete_by_id(job_id).exec(&txn).await?;
         if !internal_table_ids.is_empty() {
             Object::delete_many()
@@ -473,8 +557,28 @@ impl CatalogController {
         if let Some(source_id) = associated_source_id {
             Object::delete_by_id(source_id).exec(&txn).await?;
         }
+
+        for tx in inner
+            .creating_table_finish_notifier
+            .remove(&job_id)
+            .into_iter()
+            .flatten()
+        {
+            let err = if is_cancelled {
+                MetaError::cancelled(format!("streaming job {job_id} is cancelled"))
+            } else {
+                MetaError::catalog_id_not_found(
+                    "stream job",
+                    format!("streaming job {job_id} failed"),
+                )
+            };
+            let _ = tx.send(Err(err));
+        }
         txn.commit().await?;
 
+        let _version = self
+            .notify_frontend(Operation::Delete, relation_group)
+            .await;
         Ok(true)
     }
 
@@ -625,9 +729,9 @@ impl CatalogController {
     pub async fn finish_streaming_job(
         &self,
         job_id: ObjectId,
-        replace_table_job_info: Option<(crate::manager::StreamingJob, Vec<MergeUpdate>, u32)>,
-    ) -> MetaResult<NotificationVersion> {
-        let inner = self.inner.write().await;
+        replace_table_job_info: Option<ReplaceTablePlan>,
+    ) -> MetaResult<()> {
+        let mut inner = self.inner.write().await;
         let txn = inner.db.begin().await?;
 
         let job_type = Object::find_by_id(job_id)
@@ -756,7 +860,12 @@ impl CatalogController {
         let fragment_mapping = get_fragment_mappings(&txn, job_id).await?;
 
         let replace_table_mapping_update = match replace_table_job_info {
-            Some((streaming_job, merge_updates, dummy_id)) => {
+            Some(ReplaceTablePlan {
+                streaming_job,
+                merge_updates,
+                dummy_id,
+                ..
+            }) => {
                 let incoming_sink_id = job_id;
 
                 let (relations, fragment_mapping) = Self::finish_replace_streaming_job_inner(
@@ -765,6 +874,7 @@ impl CatalogController {
                     None,
                     Some(incoming_sink_id as _),
                     None,
+                    vec![],
                     &txn,
                     streaming_job,
                 )
@@ -782,7 +892,7 @@ impl CatalogController {
 
         let mut version = self
             .notify_frontend(
-                NotificationOperation::Add,
+                NotificationOperation::Update,
                 NotificationInfo::RelationGroup(PbRelationGroup { relations }),
             )
             .await;
@@ -797,8 +907,13 @@ impl CatalogController {
                 )
                 .await;
         }
+        if let Some(txs) = inner.creating_table_finish_notifier.remove(&job_id) {
+            for tx in txs {
+                let _ = tx.send(Ok(version));
+            }
+        }
 
-        Ok(version)
+        Ok(())
     }
 
     pub async fn finish_replace_streaming_job(
@@ -809,6 +924,7 @@ impl CatalogController {
         table_col_index_mapping: Option<ColIndexMapping>,
         creating_sink_id: Option<SinkId>,
         dropping_sink_id: Option<SinkId>,
+        updated_sink_catalogs: Vec<SinkId>,
     ) -> MetaResult<NotificationVersion> {
         let inner = self.inner.write().await;
         let txn = inner.db.begin().await?;
@@ -819,6 +935,7 @@ impl CatalogController {
             table_col_index_mapping,
             creating_sink_id,
             dropping_sink_id,
+            updated_sink_catalogs,
             &txn,
             streaming_job,
         )
@@ -849,6 +966,7 @@ impl CatalogController {
         table_col_index_mapping: Option<ColIndexMapping>,
         creating_sink_id: Option<SinkId>,
         dropping_sink_id: Option<SinkId>,
+        updated_sink_catalogs: Vec<SinkId>,
         txn: &DatabaseTransaction,
         streaming_job: StreamingJob,
     ) -> MetaResult<(Vec<Relation>, Vec<PbFragmentWorkerSlotMapping>)> {
@@ -859,6 +977,25 @@ impl CatalogController {
 
         let job_id = table.id as ObjectId;
 
+        let original_table_catalogs = Table::find_by_id(job_id)
+            .select_only()
+            .columns([table::Column::Columns])
+            .into_tuple::<ColumnCatalogArray>()
+            .one(txn)
+            .await?
+            .ok_or_else(|| MetaError::catalog_id_not_found("table", job_id))?;
+
+        // For sinks created in earlier versions, we need to set the original_target_columns.
+        for sink_id in updated_sink_catalogs {
+            sink::ActiveModel {
+                sink_id: Set(sink_id as _),
+                original_target_columns: Set(original_table_catalogs.clone()),
+                ..Default::default()
+            }
+            .update(txn)
+            .await?;
+        }
+
         let mut table = table::ActiveModel::from(table);
         let mut incoming_sinks = table.incoming_sinks.as_ref().inner_ref().clone();
         if let Some(sink_id) = creating_sink_id {
@@ -1077,12 +1214,23 @@ impl CatalogController {
         let inner = self.inner.read().await;
         let txn = inner.db.begin().await?;
 
-        let source = Source::find_by_id(source_id)
+        {
+            let active_source = source::ActiveModel {
+                source_id: Set(source_id),
+                rate_limit: Set(rate_limit.map(|v| v as i32)),
+                ..Default::default()
+            };
+            active_source.update(&txn).await?;
+        }
+
+        let (source, obj) = Source::find_by_id(source_id)
+            .find_also_related(Object)
             .one(&txn)
             .await?
             .ok_or_else(|| {
                 MetaError::catalog_id_not_found(ObjectType::Source.as_str(), source_id)
             })?;
+
         let streaming_job_ids: Vec<ObjectId> =
             if let Some(table_id) = source.optional_associated_table_id {
                 vec![table_id]
@@ -1158,6 +1306,19 @@ impl CatalogController {
 
         txn.commit().await?;
 
+        let relation_info = PbRelationInfo::Source(ObjectModel(source, obj.unwrap()).into());
+        let relation = PbRelation {
+            relation_info: Some(relation_info),
+        };
+        let _version = self
+            .notify_frontend(
+                NotificationOperation::Update,
+                NotificationInfo::RelationGroup(PbRelationGroup {
+                    relations: vec![relation],
+                }),
+            )
+            .await;
+
         Ok(fragment_actors)
     }
 
@@ -1271,8 +1432,6 @@ impl CatalogController {
 
         let txn = inner.db.begin().await?;
 
-        let parallel_unit_to_worker = get_parallel_unit_to_worker_map(&txn).await?;
-
         let mut fragment_mapping_to_notify = vec![];
 
         // for assert only
@@ -1313,11 +1472,7 @@ impl CatalogController {
                     expr_context,
                     ..
                 },
-                // actor_status
-                PbActorStatus {
-                    parallel_unit,
-                    state: _,
-                },
+                actor_status,
             ) in newly_created_actors
             {
                 let mut actor_upstreams = BTreeMap::<FragmentId, BTreeSet<ActorId>>::new();
@@ -1354,7 +1509,6 @@ impl CatalogController {
                 );
 
                 let actor_upstreams = ActorUpstreamActors(actor_upstreams);
-                let parallel_unit = parallel_unit.unwrap();
 
                 let splits = actor_splits
                     .get(&actor_id)
@@ -1365,8 +1519,7 @@ impl CatalogController {
                     fragment_id: Set(fragment_id as _),
                     status: Set(ActorStatus::Running),
                     splits: Set(splits.map(|splits| (&PbConnectorSplits { splits }).into())),
-                    parallel_unit_id: Set(parallel_unit.id as _),
-                    worker_id: Set(parallel_unit.worker_node_id as _),
+                    worker_id: Set(actor_status.worker_id() as _),
                     upstream_actor_ids: Set(actor_upstreams),
                     vnode_bitmap: Set(vnode_bitmap.as_ref().map(|bitmap| bitmap.into())),
                     expr_context: Set(expr_context.as_ref().unwrap().into()),
@@ -1421,41 +1574,24 @@ impl CatalogController {
                 .await?
                 .ok_or_else(|| MetaError::catalog_id_not_found("fragment", fragment_id))?;
 
-            let fragment_actors = fragment.find_related(Actor).all(&txn).await?;
-
-            let mut actor_to_parallel_unit = HashMap::with_capacity(fragment_actors.len());
-            let mut actor_to_vnode_bitmap = HashMap::with_capacity(fragment_actors.len());
-            for actor in &fragment_actors {
-                actor_to_parallel_unit.insert(actor.actor_id as u32, actor.parallel_unit_id as _);
-                if let Some(vnode_bitmap) = &actor.vnode_bitmap {
-                    let bitmap = Bitmap::from(&vnode_bitmap.to_protobuf());
-                    actor_to_vnode_bitmap.insert(actor.actor_id as u32, bitmap);
-                }
-            }
-
-            let vnode_mapping = if actor_to_vnode_bitmap.is_empty() {
-                let parallel_unit = *actor_to_parallel_unit.values().exactly_one().unwrap();
-                ParallelUnitMapping::new_single(parallel_unit as ParallelUnitId)
-            } else {
-                // Generate the parallel unit mapping from the fragment's actor bitmaps.
-                assert_eq!(actor_to_vnode_bitmap.len(), actor_to_parallel_unit.len());
-                ActorMapping::from_bitmaps(&actor_to_vnode_bitmap)
-                    .to_parallel_unit(&actor_to_parallel_unit)
-            }
-            .to_protobuf();
-
-            let mut fragment = fragment.into_active_model();
-            fragment.vnode_mapping = Set((&vnode_mapping).into());
-            fragment.update(&txn).await?;
-
-            let worker_slot_mapping = ParallelUnitMapping::from_protobuf(&vnode_mapping)
-                .to_worker_slot(&parallel_unit_to_worker)?
-                .to_protobuf();
+            let job_actors = fragment
+                .find_related(Actor)
+                .all(&txn)
+                .await?
+                .into_iter()
+                .map(|actor| {
+                    (
+                        fragment_id,
+                        fragment.distribution_type,
+                        actor.actor_id,
+                        actor.vnode_bitmap,
+                        actor.worker_id,
+                        actor.status,
+                    )
+                })
+                .collect_vec();
 
-            fragment_mapping_to_notify.push(FragmentWorkerSlotMapping {
-                fragment_id: fragment_id as u32,
-                mapping: Some(worker_slot_mapping),
-            });
+            fragment_mapping_to_notify.extend(rebuild_fragment_mapping_from_actors(job_actors));
 
             // for downstream and upstream
             let removed_actor_ids: HashSet<_> = removed_actors
diff --git a/src/meta/src/controller/utils.rs b/src/meta/src/controller/utils.rs
index 8a8f1b2d71cda..088796b4e9713 100644
--- a/src/meta/src/controller/utils.rs
+++ b/src/meta/src/controller/utils.rs
@@ -16,6 +16,9 @@ use std::collections::{BTreeSet, HashMap, HashSet};
 
 use anyhow::anyhow;
 use itertools::Itertools;
+use risingwave_common::bitmap::Bitmap;
+use risingwave_common::hash;
+use risingwave_common::hash::{ActorMapping, WorkerSlotId, WorkerSlotMapping};
 use risingwave_meta_model_migration::WithQuery;
 use risingwave_meta_model_v2::actor::ActorStatus;
 use risingwave_meta_model_v2::fragment::DistributionType;
@@ -24,11 +27,17 @@ use risingwave_meta_model_v2::prelude::*;
 use risingwave_meta_model_v2::{
     actor, actor_dispatcher, connection, database, fragment, function, index, object,
     object_dependency, schema, secret, sink, source, subscription, table, user, user_privilege,
-    view, worker_property, ActorId, DataTypeArray, DatabaseId, FragmentId, FragmentVnodeMapping,
-    I32Array, ObjectId, PrivilegeId, SchemaId, SourceId, StreamNode, UserId, WorkerId,
+    view, ActorId, DataTypeArray, DatabaseId, FragmentId, I32Array, ObjectId, PrivilegeId,
+    SchemaId, SourceId, StreamNode, UserId, VnodeBitmap, WorkerId,
+};
+use risingwave_pb::catalog::{
+    PbConnection, PbFunction, PbIndex, PbSecret, PbSink, PbSource, PbSubscription, PbTable, PbView,
+};
+use risingwave_pb::meta::relation::PbRelationInfo;
+use risingwave_pb::meta::subscribe_response::Info as NotificationInfo;
+use risingwave_pb::meta::{
+    FragmentWorkerSlotMapping, PbFragmentWorkerSlotMapping, PbRelation, PbRelationGroup,
 };
-use risingwave_pb::catalog::{PbConnection, PbFunction, PbSecret, PbSubscription};
-use risingwave_pb::meta::{PbFragmentParallelUnitMapping, PbFragmentWorkerSlotMapping};
 use risingwave_pb::stream_plan::stream_node::NodeBody;
 use risingwave_pb::stream_plan::{PbFragmentTypeFlag, PbStreamNode, StreamSource};
 use risingwave_pb::user::grant_privilege::{PbAction, PbActionWithGrantOption, PbObject};
@@ -42,9 +51,7 @@ use sea_orm::{
     Order, PaginatorTrait, QueryFilter, QuerySelect, RelationTrait, Statement,
 };
 
-use crate::controller::catalog::CatalogController;
 use crate::{MetaError, MetaResult};
-
 /// This function will construct a query using recursive cte to find all objects[(id, `obj_type`)] that are used by the given object.
 ///
 /// # Examples
@@ -242,7 +249,7 @@ pub struct PartialFragmentStateTables {
 pub struct PartialActorLocation {
     pub actor_id: ActorId,
     pub fragment_id: FragmentId,
-    pub parallel_unit_id: i32,
+    pub worker_id: WorkerId,
     pub status: ActorStatus,
 }
 
@@ -842,46 +849,127 @@ pub async fn get_fragment_mappings<C>(
 where
     C: ConnectionTrait,
 {
-    let parallel_unit_to_worker = get_parallel_unit_to_worker_map(db).await?;
-
-    let fragment_mappings: Vec<(FragmentId, FragmentVnodeMapping)> = Fragment::find()
+    let job_actors: Vec<(
+        FragmentId,
+        DistributionType,
+        ActorId,
+        Option<VnodeBitmap>,
+        WorkerId,
+        ActorStatus,
+    )> = Actor::find()
         .select_only()
-        .columns([fragment::Column::FragmentId, fragment::Column::VnodeMapping])
+        .columns([
+            fragment::Column::FragmentId,
+            fragment::Column::DistributionType,
+        ])
+        .columns([
+            actor::Column::ActorId,
+            actor::Column::VnodeBitmap,
+            actor::Column::WorkerId,
+            actor::Column::Status,
+        ])
+        .join(JoinType::InnerJoin, actor::Relation::Fragment.def())
         .filter(fragment::Column::JobId.eq(job_id))
         .into_tuple()
         .all(db)
         .await?;
 
-    CatalogController::convert_fragment_mappings(fragment_mappings, &parallel_unit_to_worker)
+    Ok(rebuild_fragment_mapping_from_actors(job_actors))
 }
 
-/// `get_fragment_mappings_by_jobs` returns the fragment vnode mappings of the given job list.
-pub async fn get_fragment_mappings_by_jobs<C>(
+pub fn rebuild_fragment_mapping_from_actors(
+    job_actors: Vec<(
+        FragmentId,
+        DistributionType,
+        ActorId,
+        Option<VnodeBitmap>,
+        WorkerId,
+        ActorStatus,
+    )>,
+) -> Vec<FragmentWorkerSlotMapping> {
+    let mut all_actor_locations = HashMap::new();
+    let mut actor_bitmaps = HashMap::new();
+    let mut fragment_actors = HashMap::new();
+    let mut fragment_dist = HashMap::new();
+
+    for (fragment_id, dist, actor_id, bitmap, worker_id, actor_status) in job_actors {
+        if actor_status == ActorStatus::Inactive {
+            continue;
+        }
+
+        all_actor_locations
+            .entry(fragment_id)
+            .or_insert(HashMap::new())
+            .insert(actor_id as hash::ActorId, worker_id as u32);
+        actor_bitmaps.insert(actor_id, bitmap);
+        fragment_actors
+            .entry(fragment_id)
+            .or_insert_with(Vec::new)
+            .push(actor_id);
+        fragment_dist.insert(fragment_id, dist);
+    }
+
+    let mut result = vec![];
+    for (fragment_id, dist) in fragment_dist {
+        let mut actor_locations = all_actor_locations.remove(&fragment_id).unwrap();
+        let fragment_worker_slot_mapping = match dist {
+            DistributionType::Single => {
+                let actor = fragment_actors
+                    .remove(&fragment_id)
+                    .unwrap()
+                    .into_iter()
+                    .exactly_one()
+                    .unwrap() as hash::ActorId;
+                let actor_location = actor_locations.remove(&actor).unwrap();
+
+                WorkerSlotMapping::new_single(WorkerSlotId::new(actor_location, 0))
+            }
+            DistributionType::Hash => {
+                let actors = fragment_actors.remove(&fragment_id).unwrap();
+
+                let all_actor_bitmaps: HashMap<_, _> = actors
+                    .iter()
+                    .map(|actor_id| {
+                        let vnode_bitmap = actor_bitmaps
+                            .remove(actor_id)
+                            .flatten()
+                            .expect("actor bitmap shouldn't be none in hash fragment");
+
+                        let bitmap = Bitmap::from(&vnode_bitmap.to_protobuf());
+                        (*actor_id as hash::ActorId, bitmap)
+                    })
+                    .collect();
+
+                let actor_mapping = ActorMapping::from_bitmaps(&all_actor_bitmaps);
+
+                actor_mapping.to_worker_slot(&actor_locations)
+            }
+        };
+
+        result.push(PbFragmentWorkerSlotMapping {
+            fragment_id: fragment_id as u32,
+            mapping: Some(fragment_worker_slot_mapping.to_protobuf()),
+        })
+    }
+    result
+}
+
+pub async fn get_fragment_ids_by_jobs<C>(
     db: &C,
     job_ids: Vec<ObjectId>,
-) -> MetaResult<Vec<PbFragmentParallelUnitMapping>>
+) -> MetaResult<Vec<FragmentId>>
 where
     C: ConnectionTrait,
 {
-    if job_ids.is_empty() {
-        return Ok(vec![]);
-    }
-
-    let fragment_mappings: Vec<(FragmentId, FragmentVnodeMapping)> = Fragment::find()
+    let fragment_ids: Vec<FragmentId> = Fragment::find()
         .select_only()
-        .columns([fragment::Column::FragmentId, fragment::Column::VnodeMapping])
+        .column(fragment::Column::FragmentId)
         .filter(fragment::Column::JobId.is_in(job_ids))
         .into_tuple()
         .all(db)
         .await?;
 
-    Ok(fragment_mappings
-        .into_iter()
-        .map(|(fragment_id, mapping)| PbFragmentParallelUnitMapping {
-            fragment_id: fragment_id as _,
-            mapping: Some(mapping.to_protobuf()),
-        })
-        .collect())
+    Ok(fragment_ids)
 }
 
 /// `get_fragment_actor_ids` returns the fragment actor ids of the given fragments.
@@ -982,29 +1070,70 @@ where
     ))
 }
 
-pub(crate) async fn get_parallel_unit_to_worker_map<C>(db: &C) -> MetaResult<HashMap<u32, u32>>
-where
-    C: ConnectionTrait,
-{
-    let worker_parallel_units = WorkerProperty::find()
-        .select_only()
-        .columns([
-            worker_property::Column::WorkerId,
-            worker_property::Column::ParallelUnitIds,
-        ])
-        .into_tuple::<(WorkerId, I32Array)>()
-        .all(db)
-        .await?;
-
-    let parallel_unit_to_worker = worker_parallel_units
-        .into_iter()
-        .flat_map(|(worker_id, parallel_unit_ids)| {
-            parallel_unit_ids
-                .into_inner()
-                .into_iter()
-                .map(move |parallel_unit_id| (parallel_unit_id as u32, worker_id as u32))
-        })
-        .collect::<HashMap<_, _>>();
-
-    Ok(parallel_unit_to_worker)
+pub(crate) fn build_relation_group(relation_objects: Vec<PartialObject>) -> NotificationInfo {
+    let mut relations = vec![];
+    for obj in relation_objects {
+        match obj.obj_type {
+            ObjectType::Table => relations.push(PbRelation {
+                relation_info: Some(PbRelationInfo::Table(PbTable {
+                    id: obj.oid as _,
+                    schema_id: obj.schema_id.unwrap() as _,
+                    database_id: obj.database_id.unwrap() as _,
+                    ..Default::default()
+                })),
+            }),
+            ObjectType::Source => relations.push(PbRelation {
+                relation_info: Some(PbRelationInfo::Source(PbSource {
+                    id: obj.oid as _,
+                    schema_id: obj.schema_id.unwrap() as _,
+                    database_id: obj.database_id.unwrap() as _,
+                    ..Default::default()
+                })),
+            }),
+            ObjectType::Sink => relations.push(PbRelation {
+                relation_info: Some(PbRelationInfo::Sink(PbSink {
+                    id: obj.oid as _,
+                    schema_id: obj.schema_id.unwrap() as _,
+                    database_id: obj.database_id.unwrap() as _,
+                    ..Default::default()
+                })),
+            }),
+            ObjectType::Subscription => relations.push(PbRelation {
+                relation_info: Some(PbRelationInfo::Subscription(PbSubscription {
+                    id: obj.oid as _,
+                    schema_id: obj.schema_id.unwrap() as _,
+                    database_id: obj.database_id.unwrap() as _,
+                    ..Default::default()
+                })),
+            }),
+            ObjectType::View => relations.push(PbRelation {
+                relation_info: Some(PbRelationInfo::View(PbView {
+                    id: obj.oid as _,
+                    schema_id: obj.schema_id.unwrap() as _,
+                    database_id: obj.database_id.unwrap() as _,
+                    ..Default::default()
+                })),
+            }),
+            ObjectType::Index => {
+                relations.push(PbRelation {
+                    relation_info: Some(PbRelationInfo::Index(PbIndex {
+                        id: obj.oid as _,
+                        schema_id: obj.schema_id.unwrap() as _,
+                        database_id: obj.database_id.unwrap() as _,
+                        ..Default::default()
+                    })),
+                });
+                relations.push(PbRelation {
+                    relation_info: Some(PbRelationInfo::Table(PbTable {
+                        id: obj.oid as _,
+                        schema_id: obj.schema_id.unwrap() as _,
+                        database_id: obj.database_id.unwrap() as _,
+                        ..Default::default()
+                    })),
+                });
+            }
+            _ => unreachable!("only relations will be dropped."),
+        }
+    }
+    NotificationInfo::RelationGroup(PbRelationGroup { relations })
 }
diff --git a/src/meta/src/error.rs b/src/meta/src/error.rs
index 9d3d558ac4839..8aeaed2f9c5a8 100644
--- a/src/meta/src/error.rs
+++ b/src/meta/src/error.rs
@@ -13,7 +13,6 @@
 // limitations under the License.
 
 use risingwave_common::error::BoxedError;
-use risingwave_common::hash::ParallelUnitError;
 use risingwave_common::session_config::SessionConfigError;
 use risingwave_connector::error::ConnectorError;
 use risingwave_connector::sink::SinkError;
@@ -126,13 +125,6 @@ pub enum MetaErrorInner {
     // Indicates that recovery was triggered manually.
     #[error("adhoc recovery triggered")]
     AdhocRecovery,
-
-    #[error("ParallelUnit error: {0}")]
-    ParallelUnit(
-        #[from]
-        #[backtrace]
-        ParallelUnitError,
-    ),
 }
 
 impl MetaError {
diff --git a/src/meta/src/hummock/compaction/compaction_config.rs b/src/meta/src/hummock/compaction/compaction_config.rs
index de91bf4f79ded..798500e980b63 100644
--- a/src/meta/src/hummock/compaction/compaction_config.rs
+++ b/src/meta/src/hummock/compaction/compaction_config.rs
@@ -64,7 +64,7 @@ impl CompactionConfigBuilder {
                     compaction_config::level0_overlapping_sub_level_compact_level_count(),
                 tombstone_reclaim_ratio: compaction_config::tombstone_reclaim_ratio(),
                 enable_emergency_picker: compaction_config::enable_emergency_picker(),
-                max_l0_compact_level_count: compaction_config::max_l0_compact_level_count(),
+                max_l0_compact_level_count: Some(compaction_config::max_l0_compact_level_count()),
             },
         }
     }
diff --git a/src/meta/src/hummock/compaction/mod.rs b/src/meta/src/hummock/compaction/mod.rs
index bf4b608fe59ad..2ebbab619a793 100644
--- a/src/meta/src/hummock/compaction/mod.rs
+++ b/src/meta/src/hummock/compaction/mod.rs
@@ -17,6 +17,8 @@
 pub mod compaction_config;
 mod overlap_strategy;
 use risingwave_common::catalog::{TableId, TableOption};
+use risingwave_hummock_sdk::compact_task::CompactTask;
+use risingwave_hummock_sdk::level::Levels;
 use risingwave_pb::hummock::compact_task::{self, TaskType};
 
 mod picker;
@@ -26,11 +28,12 @@ use std::fmt::{Debug, Formatter};
 use std::sync::Arc;
 
 use picker::{LevelCompactionPicker, TierCompactionPicker};
-use risingwave_hummock_sdk::{can_concat, CompactionGroupId, HummockCompactionTaskId};
+use risingwave_hummock_sdk::table_watermark::TableWatermarks;
+use risingwave_hummock_sdk::version::HummockVersionStateTableInfo;
+use risingwave_hummock_sdk::{CompactionGroupId, HummockCompactionTaskId};
 use risingwave_pb::hummock::compaction_config::CompactionMode;
-use risingwave_pb::hummock::hummock_version::Levels;
-use risingwave_pb::hummock::{CompactTask, CompactionConfig, LevelType};
-pub use selector::CompactionSelector;
+use risingwave_pb::hummock::{CompactionConfig, LevelType};
+pub use selector::{CompactionSelector, CompactionSelectorContext};
 
 use self::selector::{EmergencySelector, LocalSelectorStatistic};
 use super::check_cg_write_limit;
@@ -89,6 +92,7 @@ impl CompactStatus {
         }
     }
 
+    #[allow(clippy::too_many_arguments)]
     pub fn get_compact_task(
         &mut self,
         levels: &Levels,
@@ -97,38 +101,44 @@ impl CompactStatus {
         group: &CompactionGroup,
         stats: &mut LocalSelectorStatistic,
         selector: &mut Box<dyn CompactionSelector>,
-        table_id_to_options: HashMap<u32, TableOption>,
+        table_id_to_options: &HashMap<u32, TableOption>,
         developer_config: Arc<CompactionDeveloperConfig>,
+        table_watermarks: &HashMap<TableId, Arc<TableWatermarks>>,
+        state_table_info: &HummockVersionStateTableInfo,
     ) -> Option<CompactionTask> {
-        // When we compact the files, we must make the result of compaction meet the following
-        // conditions, for any user key, the epoch of it in the file existing in the lower
-        // layer must be larger.
-        if let Some(task) = selector.pick_compaction(
-            task_id,
+        let selector_context = CompactionSelectorContext {
             group,
             levels,
             member_table_ids,
-            &mut self.level_handlers,
-            stats,
-            table_id_to_options.clone(),
-            developer_config.clone(),
-        ) {
+            level_handlers: &mut self.level_handlers,
+            selector_stats: stats,
+            table_id_to_options,
+            developer_config: developer_config.clone(),
+            table_watermarks,
+            state_table_info,
+        };
+        // When we compact the files, we must make the result of compaction meet the following
+        // conditions, for any user key, the epoch of it in the file existing in the lower
+        // layer must be larger.
+        if let Some(task) = selector.pick_compaction(task_id, selector_context) {
             return Some(task);
         } else {
             let compaction_group_config = &group.compaction_config;
             if check_cg_write_limit(levels, compaction_group_config.as_ref()).is_write_stop()
                 && compaction_group_config.enable_emergency_picker
             {
-                return EmergencySelector::default().pick_compaction(
-                    task_id,
+                let selector_context = CompactionSelectorContext {
                     group,
                     levels,
                     member_table_ids,
-                    &mut self.level_handlers,
-                    stats,
+                    level_handlers: &mut self.level_handlers,
+                    selector_stats: stats,
                     table_id_to_options,
                     developer_config,
-                );
+                    table_watermarks,
+                    state_table_info,
+                };
+                return EmergencySelector::default().pick_compaction(task_id, selector_context);
             }
         }
 
@@ -136,15 +146,11 @@ impl CompactStatus {
     }
 
     pub fn is_trivial_move_task(task: &CompactTask) -> bool {
-        if task.task_type() != TaskType::Dynamic && task.task_type() != TaskType::Emergency {
+        if task.task_type != TaskType::Dynamic && task.task_type != TaskType::Emergency {
             return false;
         }
 
-        if task.input_ssts.len() == 1 {
-            return task.input_ssts[0].level_idx == 0
-                && can_concat(&task.input_ssts[0].table_infos);
-        } else if task.input_ssts.len() != 2
-            || task.input_ssts[0].level_type() != LevelType::Nonoverlapping
+        if task.input_ssts.len() != 2 || task.input_ssts[0].level_type != LevelType::Nonoverlapping
         {
             return false;
         }
@@ -166,6 +172,10 @@ impl CompactStatus {
     }
 
     pub fn is_trivial_reclaim(task: &CompactTask) -> bool {
+        // Currently all VnodeWatermark tasks are trivial reclaim.
+        if task.task_type == TaskType::VnodeWatermark {
+            return true;
+        }
         let exist_table_ids = HashSet::<u32>::from_iter(task.existing_table_ids.clone());
         task.input_ssts.iter().all(|level| {
             level.table_infos.iter().all(|sst| {
@@ -176,7 +186,6 @@ impl CompactStatus {
         })
     }
 
-    /// Declares a task as either succeeded, failed or canceled.
     pub fn report_compact_task(&mut self, compact_task: &CompactTask) {
         for level in &compact_task.input_ssts {
             self.level_handlers[level.level_idx as usize].remove_task(compact_task.task_id);
diff --git a/src/meta/src/hummock/compaction/overlap_strategy.rs b/src/meta/src/hummock/compaction/overlap_strategy.rs
index 1a713cc401175..9a8617d0e927b 100644
--- a/src/meta/src/hummock/compaction/overlap_strategy.rs
+++ b/src/meta/src/hummock/compaction/overlap_strategy.rs
@@ -17,9 +17,9 @@ use std::fmt::Debug;
 use std::ops::Range;
 
 use itertools::Itertools;
-use risingwave_hummock_sdk::key_range::KeyRangeCommon;
+use risingwave_hummock_sdk::key_range::{KeyRange, KeyRangeCommon};
+use risingwave_hummock_sdk::sstable_info::SstableInfo;
 use risingwave_hummock_sdk::KeyComparator;
-use risingwave_pb::hummock::{KeyRange, SstableInfo};
 
 pub trait OverlapInfo: Debug {
     fn check_overlap(&self, a: &SstableInfo) -> bool;
@@ -85,18 +85,14 @@ impl OverlapInfo for RangeOverlapInfo {
         match self.target_range.as_ref() {
             Some(key_range) => {
                 let overlap_begin = others.partition_point(|table_status| {
-                    table_status
-                        .key_range
-                        .as_ref()
-                        .unwrap()
-                        .compare_right_with(&key_range.left)
+                    table_status.key_range.compare_right_with(&key_range.left)
                         == cmp::Ordering::Less
                 });
                 if overlap_begin >= others.len() {
                     return overlap_begin..overlap_begin;
                 }
                 let overlap_end = others.partition_point(|table_status| {
-                    key_range.compare_right_with(&table_status.key_range.as_ref().unwrap().left)
+                    key_range.compare_right_with(&table_status.key_range.left)
                         != cmp::Ordering::Less
                 });
                 overlap_begin..overlap_end
@@ -110,7 +106,7 @@ impl OverlapInfo for RangeOverlapInfo {
             Some(key_range) => {
                 let overlap_begin = others.partition_point(|table_status| {
                     KeyComparator::compare_encoded_full_key(
-                        &table_status.key_range.as_ref().unwrap().left,
+                        &table_status.key_range.left,
                         &key_range.left,
                     ) == cmp::Ordering::Less
                 });
@@ -119,9 +115,7 @@ impl OverlapInfo for RangeOverlapInfo {
                 }
                 let mut overlap_end = overlap_begin;
                 for table in &others[overlap_begin..] {
-                    if key_range.compare_right_with(&table.key_range.as_ref().unwrap().right)
-                        == cmp::Ordering::Less
-                    {
+                    if key_range.compare_right_with(&table.key_range.right) == cmp::Ordering::Less {
                         break;
                     }
                     overlap_end += 1;
@@ -133,7 +127,7 @@ impl OverlapInfo for RangeOverlapInfo {
     }
 
     fn update(&mut self, table: &SstableInfo) {
-        let other = table.key_range.as_ref().unwrap();
+        let other = &table.key_range;
         if let Some(range) = self.target_range.as_mut() {
             range.full_key_extend(other);
             return;
@@ -147,8 +141,7 @@ pub struct RangeOverlapStrategy {}
 
 impl OverlapStrategy for RangeOverlapStrategy {
     fn check_overlap(&self, a: &SstableInfo, b: &SstableInfo) -> bool {
-        let key_range = a.key_range.as_ref().unwrap();
-        check_table_overlap(key_range, b)
+        check_table_overlap(&a.key_range, b)
     }
 
     fn create_overlap_info(&self) -> Box<dyn OverlapInfo> {
@@ -157,6 +150,5 @@ impl OverlapStrategy for RangeOverlapStrategy {
 }
 
 fn check_table_overlap(key_range: &KeyRange, table: &SstableInfo) -> bool {
-    let other = table.key_range.as_ref().unwrap();
-    key_range.sstable_overlap(other)
+    key_range.sstable_overlap(&table.key_range)
 }
diff --git a/src/meta/src/hummock/compaction/picker/base_level_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/base_level_compaction_picker.rs
index bf05afc6c6e88..52ea3017d49a2 100644
--- a/src/meta/src/hummock/compaction/picker/base_level_compaction_picker.rs
+++ b/src/meta/src/hummock/compaction/picker/base_level_compaction_picker.rs
@@ -16,9 +16,9 @@ use std::cell::RefCell;
 use std::sync::Arc;
 
 use itertools::Itertools;
-use risingwave_hummock_sdk::compaction_group::hummock_version_ext::HummockLevelsExt;
-use risingwave_pb::hummock::hummock_version::Levels;
-use risingwave_pb::hummock::{CompactionConfig, InputLevel, Level, LevelType, OverlappingLevel};
+use risingwave_common::config::default::compaction_config;
+use risingwave_hummock_sdk::level::{InputLevel, Level, Levels, OverlappingLevel};
+use risingwave_pb::hummock::{CompactionConfig, LevelType};
 
 use super::min_overlap_compaction_picker::NonOverlapSubLevelPicker;
 use super::{
@@ -47,11 +47,11 @@ impl CompactionPicker for LevelCompactionPicker {
         level_handlers: &[LevelHandler],
         stats: &mut LocalPickerStatistic,
     ) -> Option<CompactionInput> {
-        let l0 = levels.l0.as_ref().unwrap();
+        let l0 = &levels.l0;
         if l0.sub_levels.is_empty() {
             return None;
         }
-        if l0.sub_levels[0].level_type != LevelType::Nonoverlapping as i32
+        if l0.sub_levels[0].level_type != LevelType::Nonoverlapping
             && l0.sub_levels[0].table_infos.len() > 1
         {
             stats.skip_by_overlapping += 1;
@@ -166,7 +166,9 @@ impl LevelCompactionPicker {
             self.config.level0_max_compact_file_number,
             overlap_strategy.clone(),
             self.developer_config.enable_check_task_level_overlap,
-            self.config.max_l0_compact_level_count as usize,
+            self.config
+                .max_l0_compact_level_count
+                .unwrap_or(compaction_config::max_l0_compact_level_count()) as usize,
         );
 
         let mut max_vnode_partition_idx = 0;
@@ -231,7 +233,7 @@ impl LevelCompactionPicker {
                 .into_iter()
                 .map(|table_infos| InputLevel {
                     level_idx: 0,
-                    level_type: LevelType::Nonoverlapping as i32,
+                    level_type: LevelType::Nonoverlapping,
                     table_infos,
                 })
                 .collect_vec();
@@ -313,11 +315,11 @@ pub mod tests {
             ],
         );
         let mut levels = Levels {
-            l0: Some(OverlappingLevel {
+            l0: OverlappingLevel {
                 total_file_size: l0.total_file_size,
                 uncompressed_file_size: l0.total_file_size,
                 sub_levels: vec![l0],
-            }),
+            },
             levels: vec![generate_level(
                 1,
                 vec![
@@ -335,8 +337,8 @@ pub mod tests {
             .pick_compaction(&levels, &levels_handler, &mut local_stats)
             .unwrap();
         assert_eq!(ret.input_levels[0].table_infos.len(), 1);
-        assert_eq!(ret.input_levels[0].table_infos[0].get_sst_id(), 4);
-        assert_eq!(ret.input_levels[1].table_infos[0].get_sst_id(), 1);
+        assert_eq!(ret.input_levels[0].table_infos[0].sst_id, 4);
+        assert_eq!(ret.input_levels[1].table_infos[0].sst_id, 1);
 
         ret.add_pending_task(0, &mut levels_handler);
         {
@@ -347,14 +349,14 @@ pub mod tests {
                 .unwrap();
 
             assert_eq!(ret2.input_levels[0].table_infos.len(), 1);
-            assert_eq!(ret2.input_levels[0].table_infos[0].get_sst_id(), 6);
-            assert_eq!(ret2.input_levels[1].table_infos[0].get_sst_id(), 5);
+            assert_eq!(ret2.input_levels[0].table_infos[0].sst_id, 6);
+            assert_eq!(ret2.input_levels[1].table_infos[0].sst_id, 5);
         }
 
-        levels.l0.as_mut().unwrap().sub_levels[0]
+        levels.l0.sub_levels[0]
             .table_infos
-            .retain(|table| table.get_sst_id() != 4);
-        levels.l0.as_mut().unwrap().total_file_size -= ret.input_levels[0].table_infos[0].file_size;
+            .retain(|table| table.sst_id != 4);
+        levels.l0.total_file_size -= ret.input_levels[0].table_infos[0].file_size;
 
         levels_handler[0].remove_task(0);
         levels_handler[1].remove_task(0);
@@ -363,11 +365,11 @@ pub mod tests {
             .pick_compaction(&levels, &levels_handler, &mut local_stats)
             .unwrap();
         assert_eq!(ret.input_levels.len(), 3);
-        assert_eq!(ret.input_levels[0].table_infos[0].get_sst_id(), 6);
-        assert_eq!(ret.input_levels[1].table_infos[0].get_sst_id(), 5);
+        assert_eq!(ret.input_levels[0].table_infos[0].sst_id, 6);
+        assert_eq!(ret.input_levels[1].table_infos[0].sst_id, 5);
         assert_eq!(ret.input_levels[2].table_infos.len(), 2);
-        assert_eq!(ret.input_levels[2].table_infos[0].get_sst_id(), 3);
-        assert_eq!(ret.input_levels[2].table_infos[1].get_sst_id(), 2);
+        assert_eq!(ret.input_levels[2].table_infos[0].sst_id, 3);
+        assert_eq!(ret.input_levels[2].table_infos[1].sst_id, 2);
         ret.add_pending_task(1, &mut levels_handler);
 
         let mut local_stats = LocalPickerStatistic::default();
@@ -384,8 +386,8 @@ pub mod tests {
             .pick_compaction(&levels, &levels_handler, &mut local_stats)
             .unwrap();
         assert_eq!(ret.input_levels.len(), 3);
-        assert_eq!(ret.input_levels[0].table_infos[0].get_sst_id(), 6);
-        assert_eq!(ret.input_levels[1].table_infos[0].get_sst_id(), 5);
+        assert_eq!(ret.input_levels[0].table_infos[0].sst_id, 6);
+        assert_eq!(ret.input_levels[1].table_infos[0].sst_id, 5);
         assert_eq!(ret.input_levels[2].table_infos.len(), 2);
     }
     #[test]
@@ -403,7 +405,7 @@ pub mod tests {
 
         let levels = vec![Level {
             level_idx: 1,
-            level_type: LevelType::Nonoverlapping as i32,
+            level_type: LevelType::Nonoverlapping,
             table_infos: vec![
                 generate_table(3, 1, 0, 50, 1),
                 generate_table(4, 1, 150, 200, 1),
@@ -413,11 +415,11 @@ pub mod tests {
         }];
         let mut levels = Levels {
             levels,
-            l0: Some(OverlappingLevel {
+            l0: OverlappingLevel {
                 sub_levels: vec![],
                 total_file_size: 0,
                 uncompressed_file_size: 0,
-            }),
+            },
             ..Default::default()
         };
         push_tables_level0_nonoverlapping(&mut levels, vec![generate_table(1, 1, 50, 140, 2)]);
@@ -444,7 +446,7 @@ pub mod tests {
             ret.input_levels[0]
                 .table_infos
                 .iter()
-                .map(|t| t.get_sst_id())
+                .map(|t| t.sst_id)
                 .collect_vec(),
             vec![1]
         );
@@ -453,7 +455,7 @@ pub mod tests {
             ret.input_levels[1]
                 .table_infos
                 .iter()
-                .map(|t| t.get_sst_id())
+                .map(|t| t.sst_id)
                 .collect_vec(),
             vec![3,]
         );
@@ -466,7 +468,7 @@ pub mod tests {
         let mut picker = create_compaction_picker_for_test();
         let levels = vec![Level {
             level_idx: 1,
-            level_type: LevelType::Nonoverlapping as i32,
+            level_type: LevelType::Nonoverlapping,
             table_infos: vec![],
             total_file_size: 0,
             sub_level_id: 0,
@@ -475,11 +477,11 @@ pub mod tests {
         }];
         let mut levels = Levels {
             levels,
-            l0: Some(OverlappingLevel {
+            l0: OverlappingLevel {
                 sub_levels: vec![],
                 total_file_size: 0,
                 uncompressed_file_size: 0,
-            }),
+            },
             ..Default::default()
         };
         push_tables_level0_nonoverlapping(
@@ -530,7 +532,7 @@ pub mod tests {
         let mut levels = Levels {
             levels: vec![Level {
                 level_idx: 1,
-                level_type: LevelType::Nonoverlapping as i32,
+                level_type: LevelType::Nonoverlapping,
                 table_infos: vec![
                     generate_table(1, 1, 100, 399, 2),
                     generate_table(2, 1, 400, 699, 2),
@@ -541,7 +543,7 @@ pub mod tests {
                 uncompressed_file_size: 900,
                 ..Default::default()
             }],
-            l0: Some(generate_l0_nonoverlapping_sublevels(vec![])),
+            l0: generate_l0_nonoverlapping_sublevels(vec![]),
             ..Default::default()
         };
         push_tables_level0_nonoverlapping(
@@ -555,11 +557,7 @@ pub mod tests {
 
         let mut levels_handler = vec![LevelHandler::new(0), LevelHandler::new(1)];
         let mut local_stats = LocalPickerStatistic::default();
-        levels_handler[0].add_pending_task(
-            1,
-            4,
-            &levels.l0.as_ref().unwrap().sub_levels[0].table_infos,
-        );
+        levels_handler[0].add_pending_task(1, 4, &levels.l0.sub_levels[0].table_infos);
         let ret = picker.pick_compaction(&levels, &levels_handler, &mut local_stats);
         // Skip this compaction because the write amplification is too large.
         assert!(ret.is_none());
@@ -578,10 +576,10 @@ pub mod tests {
         ]);
         // We can set level_type only because the input above is valid.
         for s in &mut l0.sub_levels {
-            s.level_type = LevelType::Nonoverlapping as i32;
+            s.level_type = LevelType::Nonoverlapping;
         }
         let levels = Levels {
-            l0: Some(l0),
+            l0,
             levels: vec![generate_level(1, vec![generate_table(3, 1, 0, 100000, 1)])],
             ..Default::default()
         };
@@ -603,7 +601,7 @@ pub mod tests {
         let ret = picker
             .pick_compaction(&levels, &levels_handler, &mut local_stats)
             .unwrap();
-        assert_eq!(ret.input_levels[0].table_infos[0].get_sst_id(), 7);
+        assert_eq!(ret.input_levels[0].table_infos[0].sst_id, 7);
         assert_eq!(
             3,
             ret.input_levels.iter().filter(|l| l.level_idx == 0).count()
@@ -631,7 +629,7 @@ pub mod tests {
         let ret = picker
             .pick_compaction(&levels, &levels_handler, &mut local_stats)
             .unwrap();
-        assert_eq!(ret.input_levels[0].table_infos[0].get_sst_id(), 6);
+        assert_eq!(ret.input_levels[0].table_infos[0].sst_id, 6);
         assert_eq!(
             2,
             ret.input_levels.iter().filter(|l| l.level_idx == 0).count()
@@ -658,7 +656,7 @@ pub mod tests {
         ]);
 
         let levels = Levels {
-            l0: Some(l0),
+            l0,
             levels: vec![generate_level(1, vec![generate_table(3, 1, 0, 100000, 1)])],
             ..Default::default()
         };
@@ -666,7 +664,7 @@ pub mod tests {
         let mut local_stats = LocalPickerStatistic::default();
 
         // Create a pending sub-level.
-        let pending_level = levels.l0.as_ref().unwrap().sub_levels[1].clone();
+        let pending_level = levels.l0.sub_levels[1].clone();
         assert_eq!(pending_level.sub_level_id, 1);
         let tier_task_input = CompactionInput {
             input_levels: vec![InputLevel {
@@ -724,7 +722,7 @@ pub mod tests {
         ]);
 
         let levels = Levels {
-            l0: Some(l0),
+            l0,
             levels: vec![generate_level(1, vec![generate_table(3, 1, 1, 100, 1)])],
             ..Default::default()
         };
diff --git a/src/meta/src/hummock/compaction/picker/compaction_task_validator.rs b/src/meta/src/hummock/compaction/picker/compaction_task_validator.rs
index c7dd27a6b1907..8bdb10c213d16 100644
--- a/src/meta/src/hummock/compaction/picker/compaction_task_validator.rs
+++ b/src/meta/src/hummock/compaction/picker/compaction_task_validator.rs
@@ -15,6 +15,7 @@
 use std::collections::HashMap;
 use std::sync::Arc;
 
+use risingwave_common::config::default::compaction_config;
 use risingwave_pb::hummock::CompactionConfig;
 
 use super::{CompactionInput, LocalPickerStatistic};
@@ -90,7 +91,12 @@ struct TierCompactionTaskValidationRule {
 impl CompactionTaskValidationRule for TierCompactionTaskValidationRule {
     fn validate(&self, input: &CompactionInput, stats: &mut LocalPickerStatistic) -> bool {
         if input.total_file_count >= self.config.level0_max_compact_file_number
-            || input.input_levels.len() >= self.config.max_l0_compact_level_count as usize
+            || input.input_levels.len()
+                >= self
+                    .config
+                    .max_l0_compact_level_count
+                    .unwrap_or(compaction_config::max_l0_compact_level_count())
+                    as usize
         {
             return true;
         }
@@ -124,7 +130,12 @@ impl CompactionTaskValidationRule for IntraCompactionTaskValidationRule {
     fn validate(&self, input: &CompactionInput, stats: &mut LocalPickerStatistic) -> bool {
         if (input.total_file_count >= self.config.level0_max_compact_file_number
             && input.input_levels.len() > 1)
-            || input.input_levels.len() >= self.config.max_l0_compact_level_count as usize
+            || input.input_levels.len()
+                >= self
+                    .config
+                    .max_l0_compact_level_count
+                    .unwrap_or(compaction_config::max_l0_compact_level_count())
+                    as usize
         {
             return true;
         }
@@ -172,7 +183,12 @@ struct BaseCompactionTaskValidationRule {
 impl CompactionTaskValidationRule for BaseCompactionTaskValidationRule {
     fn validate(&self, input: &CompactionInput, stats: &mut LocalPickerStatistic) -> bool {
         if input.total_file_count >= self.config.level0_max_compact_file_number
-            || input.input_levels.len() >= self.config.max_l0_compact_level_count as usize
+            || input.input_levels.len()
+                >= self
+                    .config
+                    .max_l0_compact_level_count
+                    .unwrap_or(compaction_config::max_l0_compact_level_count())
+                    as usize
         {
             return true;
         }
diff --git a/src/meta/src/hummock/compaction/picker/emergency_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/emergency_compaction_picker.rs
index 4efcca28a981c..ab91589bdd46d 100644
--- a/src/meta/src/hummock/compaction/picker/emergency_compaction_picker.rs
+++ b/src/meta/src/hummock/compaction/picker/emergency_compaction_picker.rs
@@ -14,7 +14,7 @@
 
 use std::sync::Arc;
 
-use risingwave_pb::hummock::hummock_version::Levels;
+use risingwave_hummock_sdk::level::Levels;
 use risingwave_pb::hummock::{CompactionConfig, LevelType};
 
 use super::{
@@ -51,26 +51,24 @@ impl EmergencyCompactionPicker {
         stats: &mut LocalPickerStatistic,
     ) -> Option<CompactionInput> {
         let unused_validator = Arc::new(CompactionTaskValidator::unused());
-        let l0 = levels.l0.as_ref().unwrap();
+        let l0 = &levels.l0;
         let overlapping_count = l0
             .sub_levels
             .iter()
-            .filter(|level| level.level_type == LevelType::Overlapping as i32)
+            .filter(|level| level.level_type == LevelType::Overlapping)
             .count();
         let no_overlap_count = l0
             .sub_levels
             .iter()
             .filter(|level| {
-                level.level_type == LevelType::Nonoverlapping as i32
-                    && level.vnode_partition_count == 0
+                level.level_type == LevelType::Nonoverlapping && level.vnode_partition_count == 0
             })
             .count();
         let partitioned_count = l0
             .sub_levels
             .iter()
             .filter(|level| {
-                level.level_type == LevelType::Nonoverlapping as i32
-                    && level.vnode_partition_count > 0
+                level.level_type == LevelType::Nonoverlapping && level.vnode_partition_count > 0
             })
             .count();
         // We trigger `EmergencyCompactionPicker` only when some unexpected condition cause the number of l0 levels increase and the origin strategy
@@ -102,7 +100,7 @@ impl EmergencyCompactionPicker {
                 WholeLevelCompactionPicker::new(self.config.clone(), unused_validator.clone());
 
             if let Some(ret) = intral_level_compaction_picker.pick_whole_level(
-                levels.l0.as_ref().unwrap(),
+                &levels.l0,
                 &level_handlers[0],
                 self.config.split_weight_by_vnode,
                 stats,
diff --git a/src/meta/src/hummock/compaction/picker/intra_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/intra_compaction_picker.rs
index 5cc65bd38a1c8..6f0bfced1cc03 100644
--- a/src/meta/src/hummock/compaction/picker/intra_compaction_picker.rs
+++ b/src/meta/src/hummock/compaction/picker/intra_compaction_picker.rs
@@ -14,8 +14,9 @@
 
 use std::sync::Arc;
 
-use risingwave_pb::hummock::hummock_version::Levels;
-use risingwave_pb::hummock::{CompactionConfig, InputLevel, LevelType, OverlappingLevel};
+use risingwave_common::config::default::compaction_config;
+use risingwave_hummock_sdk::level::{InputLevel, Levels, OverlappingLevel};
+use risingwave_pb::hummock::{CompactionConfig, LevelType};
 
 use super::min_overlap_compaction_picker::NonOverlapSubLevelPicker;
 use super::{
@@ -40,7 +41,7 @@ impl CompactionPicker for IntraCompactionPicker {
         level_handlers: &[LevelHandler],
         stats: &mut LocalPickerStatistic,
     ) -> Option<CompactionInput> {
-        let l0 = levels.l0.as_ref().unwrap();
+        let l0 = &levels.l0;
         if l0.sub_levels.is_empty() {
             return None;
         }
@@ -54,10 +55,33 @@ impl CompactionPicker for IntraCompactionPicker {
         if let Some(ret) =
             self.pick_whole_level(l0, &level_handlers[0], vnode_partition_count, stats)
         {
+            if ret.input_levels.len() < 2 {
+                tracing::error!(
+                    ?ret,
+                    vnode_partition_count,
+                    "pick_whole_level failed to pick enough levels"
+                );
+                return None;
+            }
+
             return Some(ret);
         }
 
-        self.pick_l0_intra(l0, &level_handlers[0], vnode_partition_count, stats)
+        if let Some(ret) = self.pick_l0_intra(l0, &level_handlers[0], vnode_partition_count, stats)
+        {
+            if ret.input_levels.len() < 2 {
+                tracing::error!(
+                    ?ret,
+                    vnode_partition_count,
+                    "pick_l0_intra failed to pick enough levels"
+                );
+                return None;
+            }
+
+            return Some(ret);
+        }
+
+        None
     }
 }
 
@@ -118,7 +142,7 @@ impl IntraCompactionPicker {
         }
 
         for (idx, level) in l0.sub_levels.iter().enumerate() {
-            if level.level_type() != LevelType::Nonoverlapping
+            if level.level_type != LevelType::Nonoverlapping
                 || level.total_file_size > self.config.sub_level_max_compaction_bytes
             {
                 continue;
@@ -144,7 +168,10 @@ impl IntraCompactionPicker {
                 self.config.level0_max_compact_file_number,
                 overlap_strategy.clone(),
                 self.developer_config.enable_check_task_level_overlap,
-                self.config.max_l0_compact_level_count as usize,
+                self.config
+                    .max_l0_compact_level_count
+                    .unwrap_or(compaction_config::max_l0_compact_level_count())
+                    as usize,
             );
 
             let l0_select_tables_vec = non_overlap_sub_level_picker
@@ -177,7 +204,7 @@ impl IntraCompactionPicker {
                     }
                     select_level_inputs.push(InputLevel {
                         level_idx: 0,
-                        level_type: LevelType::Nonoverlapping as i32,
+                        level_type: LevelType::Nonoverlapping,
                         table_infos: level_select_sst,
                     });
 
@@ -222,11 +249,11 @@ impl IntraCompactionPicker {
         let overlap_strategy = create_overlap_strategy(self.config.compaction_mode());
 
         for (idx, level) in l0.sub_levels.iter().enumerate() {
-            if level.level_type == LevelType::Overlapping as i32 || idx + 1 >= l0.sub_levels.len() {
+            if level.level_type == LevelType::Overlapping || idx + 1 >= l0.sub_levels.len() {
                 continue;
             }
 
-            if l0.sub_levels[idx + 1].level_type == LevelType::Overlapping as i32 {
+            if l0.sub_levels[idx + 1].level_type == LevelType::Overlapping {
                 continue;
             }
 
@@ -268,12 +295,12 @@ impl IntraCompactionPicker {
             let input_levels = vec![
                 InputLevel {
                     level_idx: 0,
-                    level_type: LevelType::Nonoverlapping as i32,
+                    level_type: LevelType::Nonoverlapping,
                     table_infos: vec![select_sst],
                 },
                 InputLevel {
                     level_idx: 0,
-                    level_type: LevelType::Nonoverlapping as i32,
+                    level_type: LevelType::Nonoverlapping,
                     table_infos: vec![],
                 },
             ];
@@ -317,7 +344,7 @@ impl WholeLevelCompactionPicker {
             return None;
         }
         for (idx, level) in l0.sub_levels.iter().enumerate() {
-            if level.level_type() != LevelType::Nonoverlapping
+            if level.level_type != LevelType::Nonoverlapping
                 || level.vnode_partition_count == partition_count
             {
                 continue;
@@ -357,7 +384,7 @@ impl WholeLevelCompactionPicker {
                     table_infos: next_level.table_infos.clone(),
                 });
             }
-            if !select_level_inputs.is_empty() {
+            if select_level_inputs.len() > 1 {
                 let vnode_partition_count =
                     if select_input_size > self.config.sub_level_max_compaction_bytes / 2 {
                         partition_count
@@ -390,7 +417,7 @@ impl WholeLevelCompactionPicker {
 
 #[cfg(test)]
 pub mod tests {
-    use risingwave_pb::hummock::Level;
+    use risingwave_hummock_sdk::level::Level;
 
     use super::*;
     use crate::hummock::compaction::compaction_config::CompactionConfigBuilder;
@@ -417,17 +444,17 @@ pub mod tests {
         // compacting_key_range.
         let levels = vec![Level {
             level_idx: 1,
-            level_type: LevelType::Nonoverlapping as i32,
+            level_type: LevelType::Nonoverlapping,
             table_infos: vec![],
             ..Default::default()
         }];
         let mut levels = Levels {
             levels,
-            l0: Some(OverlappingLevel {
+            l0: OverlappingLevel {
                 sub_levels: vec![],
                 total_file_size: 0,
                 uncompressed_file_size: 0,
-            }),
+            },
             ..Default::default()
         };
         push_tables_level0_nonoverlapping(
@@ -464,14 +491,14 @@ pub mod tests {
         let mut levels = Levels {
             levels: vec![Level {
                 level_idx: 1,
-                level_type: LevelType::Nonoverlapping as i32,
+                level_type: LevelType::Nonoverlapping,
                 table_infos: vec![generate_table(3, 1, 200, 300, 2)],
                 ..Default::default()
             }],
-            l0: Some(generate_l0_nonoverlapping_sublevels(vec![
+            l0: generate_l0_nonoverlapping_sublevels(vec![
                 generate_table(1, 1, 100, 210, 2),
                 generate_table(2, 1, 200, 250, 2),
-            ])),
+            ]),
             ..Default::default()
         };
         let mut levels_handler = vec![LevelHandler::new(0), LevelHandler::new(1)];
@@ -507,12 +534,12 @@ pub mod tests {
                 ],
             ]);
             let levels = Levels {
-                l0: Some(l0),
+                l0,
                 levels: vec![generate_level(1, vec![generate_table(100, 1, 0, 1000, 1)])],
                 ..Default::default()
             };
             let mut levels_handler = vec![LevelHandler::new(0), LevelHandler::new(1)];
-            levels_handler[1].add_pending_task(100, 1, levels.levels[0].get_table_infos());
+            levels_handler[1].add_pending_task(100, 1, &levels.levels[0].table_infos);
             let config = Arc::new(
                 CompactionConfigBuilder::new()
                     .level0_sub_level_compact_level_count(1)
@@ -555,12 +582,12 @@ pub mod tests {
                 ],
             ]);
             let levels = Levels {
-                l0: Some(l0),
+                l0,
                 levels: vec![generate_level(1, vec![generate_table(100, 1, 0, 1000, 1)])],
                 ..Default::default()
             };
             let mut levels_handler = vec![LevelHandler::new(0), LevelHandler::new(1)];
-            levels_handler[1].add_pending_task(100, 1, levels.levels[0].get_table_infos());
+            levels_handler[1].add_pending_task(100, 1, &levels.levels[0].table_infos);
             let config = Arc::new(
                 CompactionConfigBuilder::new()
                     .level0_sub_level_compact_level_count(1)
@@ -583,9 +610,9 @@ pub mod tests {
                 3
             );
 
-            assert_eq!(4, ret.input_levels[0].table_infos[0].get_sst_id());
-            assert_eq!(3, ret.input_levels[1].table_infos[0].get_sst_id());
-            assert_eq!(1, ret.input_levels[2].table_infos[0].get_sst_id());
+            assert_eq!(4, ret.input_levels[0].table_infos[0].sst_id);
+            assert_eq!(3, ret.input_levels[1].table_infos[0].sst_id);
+            assert_eq!(1, ret.input_levels[2].table_infos[0].sst_id);
 
             // will pick sst [2, 6, 5]
             let ret2 = picker
@@ -600,9 +627,9 @@ pub mod tests {
                 3
             );
 
-            assert_eq!(5, ret2.input_levels[0].table_infos[0].get_sst_id());
-            assert_eq!(6, ret2.input_levels[1].table_infos[0].get_sst_id());
-            assert_eq!(2, ret2.input_levels[2].table_infos[0].get_sst_id());
+            assert_eq!(5, ret2.input_levels[0].table_infos[0].sst_id);
+            assert_eq!(6, ret2.input_levels[1].table_infos[0].sst_id);
+            assert_eq!(2, ret2.input_levels[2].table_infos[0].sst_id);
         }
 
         {
@@ -626,12 +653,12 @@ pub mod tests {
                 ],
             ]);
             let levels = Levels {
-                l0: Some(l0),
+                l0,
                 levels: vec![generate_level(1, vec![generate_table(100, 1, 0, 1000, 1)])],
                 ..Default::default()
             };
             let mut levels_handler = vec![LevelHandler::new(0), LevelHandler::new(1)];
-            levels_handler[1].add_pending_task(100, 1, levels.levels[0].get_table_infos());
+            levels_handler[1].add_pending_task(100, 1, &levels.levels[0].table_infos);
             let config = Arc::new(
                 CompactionConfigBuilder::new()
                     .level0_sub_level_compact_level_count(1)
@@ -654,9 +681,9 @@ pub mod tests {
                 3
             );
 
-            assert_eq!(11, ret.input_levels[0].table_infos[0].get_sst_id());
-            assert_eq!(9, ret.input_levels[1].table_infos[0].get_sst_id());
-            assert_eq!(7, ret.input_levels[2].table_infos[0].get_sst_id());
+            assert_eq!(11, ret.input_levels[0].table_infos[0].sst_id);
+            assert_eq!(9, ret.input_levels[1].table_infos[0].sst_id);
+            assert_eq!(7, ret.input_levels[2].table_infos[0].sst_id);
 
             let ret2 = picker
                 .pick_compaction(&levels, &levels_handler, &mut local_stats)
@@ -670,9 +697,9 @@ pub mod tests {
                 3
             );
 
-            assert_eq!(5, ret2.input_levels[0].table_infos[0].get_sst_id());
-            assert_eq!(10, ret2.input_levels[1].table_infos[0].get_sst_id());
-            assert_eq!(2, ret2.input_levels[2].table_infos[0].get_sst_id());
+            assert_eq!(5, ret2.input_levels[0].table_infos[0].sst_id);
+            assert_eq!(10, ret2.input_levels[1].table_infos[0].sst_id);
+            assert_eq!(2, ret2.input_levels[2].table_infos[0].sst_id);
         }
     }
 
@@ -701,11 +728,11 @@ pub mod tests {
             generate_table(2, 1, 150, 250, 1),
         ]]);
         let levels = Levels {
-            l0: Some(l0),
+            l0,
             levels: vec![generate_level(1, vec![generate_table(100, 1, 0, 1000, 1)])],
             ..Default::default()
         };
-        levels_handler[1].add_pending_task(100, 1, levels.levels[0].get_table_infos());
+        levels_handler[1].add_pending_task(100, 1, &levels.levels[0].table_infos);
         let mut local_stats = LocalPickerStatistic::default();
         let ret = picker.pick_compaction(&levels, &levels_handler, &mut local_stats);
         assert!(ret.is_none());
@@ -721,7 +748,7 @@ pub mod tests {
             vec![generate_table(5, 1, 10, 90, 1)],
         ]);
         let mut levels = Levels {
-            l0: Some(l0),
+            l0,
             levels: vec![generate_level(1, vec![generate_table(100, 1, 0, 1000, 1)])],
             ..Default::default()
         };
@@ -730,20 +757,20 @@ pub mod tests {
             .is_none());
 
         // Cannot trivial move because latter sub-level is overlapping
-        levels.l0.as_mut().unwrap().sub_levels[0].level_type = LevelType::Nonoverlapping as i32;
-        levels.l0.as_mut().unwrap().sub_levels[1].level_type = LevelType::Overlapping as i32;
+        levels.l0.sub_levels[0].level_type = LevelType::Nonoverlapping;
+        levels.l0.sub_levels[1].level_type = LevelType::Overlapping;
         let ret = picker.pick_compaction(&levels, &levels_handler, &mut local_stats);
         assert!(ret.is_none());
 
         // Cannot trivial move because former sub-level is overlapping
-        levels.l0.as_mut().unwrap().sub_levels[0].level_type = LevelType::Overlapping as i32;
-        levels.l0.as_mut().unwrap().sub_levels[1].level_type = LevelType::Nonoverlapping as i32;
+        levels.l0.sub_levels[0].level_type = LevelType::Overlapping;
+        levels.l0.sub_levels[1].level_type = LevelType::Nonoverlapping;
         let ret = picker.pick_compaction(&levels, &levels_handler, &mut local_stats);
         assert!(ret.is_none());
 
         // trivial move
-        levels.l0.as_mut().unwrap().sub_levels[0].level_type = LevelType::Nonoverlapping as i32;
-        levels.l0.as_mut().unwrap().sub_levels[1].level_type = LevelType::Nonoverlapping as i32;
+        levels.l0.sub_levels[0].level_type = LevelType::Nonoverlapping;
+        levels.l0.sub_levels[1].level_type = LevelType::Nonoverlapping;
         let ret = picker
             .pick_compaction(&levels, &levels_handler, &mut local_stats)
             .unwrap();
@@ -818,7 +845,7 @@ pub mod tests {
         let mut local_stats = LocalPickerStatistic::default();
 
         let levels = Levels {
-            l0: Some(l0),
+            l0,
             levels: vec![generate_level(1, vec![generate_table(100, 1, 0, 1000, 1)])],
             ..Default::default()
         };
@@ -833,15 +860,11 @@ pub mod tests {
         let input = ret.as_ref().unwrap();
         assert_eq!(input.input_levels.len(), 2);
         assert_ne!(
-            levels.l0.as_ref().unwrap().get_sub_levels()[0]
-                .table_infos
-                .len(),
+            levels.l0.sub_levels[0].table_infos.len(),
             input.input_levels[0].table_infos.len()
         );
         assert_ne!(
-            levels.l0.as_ref().unwrap().get_sub_levels()[1]
-                .table_infos
-                .len(),
+            levels.l0.sub_levels[1].table_infos.len(),
             input.input_levels[1].table_infos.len()
         );
     }
diff --git a/src/meta/src/hummock/compaction/picker/manual_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/manual_compaction_picker.rs
index 2d1b2c95b20aa..98f27254ab46c 100644
--- a/src/meta/src/hummock/compaction/picker/manual_compaction_picker.rs
+++ b/src/meta/src/hummock/compaction/picker/manual_compaction_picker.rs
@@ -16,9 +16,9 @@ use std::collections::HashSet;
 use std::sync::Arc;
 
 use itertools::Itertools;
-use risingwave_hummock_sdk::compaction_group::hummock_version_ext::HummockLevelsExt;
-use risingwave_pb::hummock::hummock_version::Levels;
-use risingwave_pb::hummock::{InputLevel, Level, LevelType, OverlappingLevel, SstableInfo};
+use risingwave_hummock_sdk::level::{InputLevel, Level, Levels, OverlappingLevel};
+use risingwave_hummock_sdk::sstable_info::SstableInfo;
+use risingwave_pb::hummock::LevelType;
 
 use super::{CompactionInput, CompactionPicker, LocalPickerStatistic};
 use crate::hummock::compaction::overlap_strategy::{
@@ -114,7 +114,7 @@ impl ManualCompactionPicker {
         for l in 1..self.target_level {
             assert!(levels.levels[l - 1].table_infos.is_empty());
         }
-        let l0 = levels.l0.as_ref().unwrap();
+        let l0 = &levels.l0;
         let mut input_levels = vec![];
         let mut max_sub_level_idx = usize::MAX;
         let mut info = self.overlap_strategy.create_overlap_info();
@@ -163,7 +163,7 @@ impl ManualCompactionPicker {
         input_levels.reverse();
         input_levels.push(InputLevel {
             level_idx: self.target_level as u32,
-            level_type: LevelType::Nonoverlapping as i32,
+            level_type: LevelType::Nonoverlapping,
             table_infos: target_input_ssts,
         });
 
@@ -181,7 +181,7 @@ impl ManualCompactionPicker {
         let mut hint_sst_ids: HashSet<u64> = HashSet::new();
         hint_sst_ids.extend(self.option.sst_ids.iter());
         let tmp_sst_info = SstableInfo {
-            key_range: Some(self.option.key_range.clone()),
+            key_range: self.option.key_range.clone(),
             ..Default::default()
         };
         if self
@@ -222,7 +222,7 @@ impl CompactionPicker for ManualCompactionPicker {
     ) -> Option<CompactionInput> {
         if self.option.level == 0 {
             if !self.option.sst_ids.is_empty() {
-                return self.pick_l0_to_sub_level(levels.l0.as_ref().unwrap(), level_handlers);
+                return self.pick_l0_to_sub_level(&levels.l0, level_handlers);
             } else if self.target_level > 0 {
                 return self.pick_l0_to_base_level(levels, level_handlers);
             } else {
@@ -233,7 +233,7 @@ impl CompactionPicker for ManualCompactionPicker {
         hint_sst_ids.extend(self.option.sst_ids.iter());
         let mut tmp_sst_info = SstableInfo::default();
         let mut range_overlap_info = RangeOverlapInfo::default();
-        tmp_sst_info.key_range = Some(self.option.key_range.clone());
+        tmp_sst_info.key_range = self.option.key_range.clone();
         range_overlap_info.update(&tmp_sst_info);
         let level = self.option.level;
         let target_level = self.target_level;
@@ -271,15 +271,13 @@ impl CompactionPicker for ManualCompactionPicker {
                 .get_level(level)
                 .table_infos
                 .iter()
-                .find_position(|p| {
-                    p.get_sst_id() == select_input_ssts.first().unwrap().get_sst_id()
-                })
+                .find_position(|p| p.sst_id == select_input_ssts.first().unwrap().sst_id)
                 .unwrap();
             let (right, _) = levels
                 .get_level(level)
                 .table_infos
                 .iter()
-                .find_position(|p| p.get_sst_id() == select_input_ssts.last().unwrap().get_sst_id())
+                .find_position(|p| p.sst_id == select_input_ssts.last().unwrap().sst_id)
                 .unwrap();
             select_input_ssts = levels.get_level(level).table_infos[left..=right].to_vec();
             vec![]
@@ -328,8 +326,10 @@ impl CompactionPicker for ManualCompactionPicker {
 pub mod tests {
     use std::collections::{BTreeSet, HashMap};
 
+    use bytes::Bytes;
+    use risingwave_hummock_sdk::key_range::KeyRange;
+    use risingwave_hummock_sdk::version::HummockVersionStateTableInfo;
     use risingwave_pb::hummock::compact_task;
-    pub use risingwave_pb::hummock::KeyRange;
 
     use super::*;
     use crate::hummock::compaction::compaction_config::CompactionConfigBuilder;
@@ -341,7 +341,7 @@ pub mod tests {
     use crate::hummock::compaction::selector::{CompactionSelector, ManualCompactionSelector};
     use crate::hummock::compaction::{CompactionDeveloperConfig, LocalSelectorStatistic};
     use crate::hummock::model::CompactionGroup;
-    use crate::hummock::test_utils::iterator_test_key_of_epoch;
+    use crate::hummock::test_utils::{compaction_selector_context, iterator_test_key_of_epoch};
 
     fn clean_task_state(level_handler: &mut LevelHandler) {
         for pending_task_id in &level_handler.pending_tasks_ids() {
@@ -378,7 +378,7 @@ pub mod tests {
         let levels = vec![
             Level {
                 level_idx: 1,
-                level_type: LevelType::Nonoverlapping as i32,
+                level_type: LevelType::Nonoverlapping,
                 table_infos: vec![
                     generate_table(0, 1, 0, 100, 1),
                     generate_table(1, 1, 101, 200, 1),
@@ -388,7 +388,7 @@ pub mod tests {
             },
             Level {
                 level_idx: 2,
-                level_type: LevelType::Nonoverlapping as i32,
+                level_type: LevelType::Nonoverlapping,
                 table_infos: vec![
                     generate_table(4, 1, 0, 100, 1),
                     generate_table(5, 1, 101, 150, 1),
@@ -401,7 +401,7 @@ pub mod tests {
         ];
         let mut levels = Levels {
             levels,
-            l0: Some(generate_l0_nonoverlapping_sublevels(vec![])),
+            l0: generate_l0_nonoverlapping_sublevels(vec![]),
             ..Default::default()
         };
         let mut levels_handler = vec![
@@ -416,8 +416,8 @@ pub mod tests {
             let option = ManualCompactionOption {
                 level: 1,
                 key_range: KeyRange {
-                    left: iterator_test_key_of_epoch(1, 0, 1),
-                    right: iterator_test_key_of_epoch(1, 201, 1),
+                    left: Bytes::from(iterator_test_key_of_epoch(1, 0, 1)),
+                    right: Bytes::from(iterator_test_key_of_epoch(1, 201, 1)),
                     right_exclusive: false,
                 },
                 ..Default::default()
@@ -509,8 +509,8 @@ pub mod tests {
                 sst_ids: vec![],
                 level: 1,
                 key_range: KeyRange {
-                    left: iterator_test_key_of_epoch(1, 101, 1),
-                    right: iterator_test_key_of_epoch(1, 199, 1),
+                    left: Bytes::from(iterator_test_key_of_epoch(1, 101, 1)),
+                    right: Bytes::from(iterator_test_key_of_epoch(1, 199, 1)),
                     right_exclusive: false,
                 },
                 internal_table_id: HashSet::from([2]),
@@ -553,7 +553,7 @@ pub mod tests {
         let mut levels = vec![
             Level {
                 level_idx: 1,
-                level_type: LevelType::Nonoverlapping as i32,
+                level_type: LevelType::Nonoverlapping,
                 table_infos: vec![
                     generate_table(3, 1, 0, 100, 1),
                     generate_table(4, 2, 2000, 3000, 1),
@@ -562,7 +562,7 @@ pub mod tests {
             },
             Level {
                 level_idx: 2,
-                level_type: LevelType::Nonoverlapping as i32,
+                level_type: LevelType::Nonoverlapping,
                 table_infos: vec![
                     generate_table(1, 1, 0, 100, 1),
                     generate_table(2, 2, 2000, 3000, 1),
@@ -577,7 +577,7 @@ pub mod tests {
                 for t in &mut l.table_infos {
                     t.table_ids.clear();
                     if idx == 0 {
-                        t.table_ids.push(((t.get_sst_id() % 2) + 1) as _);
+                        t.table_ids.push(((t.sst_id % 2) + 1) as _);
                     } else {
                         t.table_ids.push(3);
                     }
@@ -586,7 +586,7 @@ pub mod tests {
         }
         let levels = Levels {
             levels,
-            l0: Some(l0),
+            l0,
             ..Default::default()
         };
 
@@ -602,7 +602,7 @@ pub mod tests {
         let l0 = generate_l0_overlapping_sublevels(vec![]);
         let levels = vec![Level {
             level_idx: 1,
-            level_type: LevelType::Nonoverlapping as i32,
+            level_type: LevelType::Nonoverlapping,
             table_infos: vec![
                 generate_table(1, 1, 0, 100, 1),
                 generate_table(2, 2, 100, 200, 1),
@@ -613,7 +613,7 @@ pub mod tests {
         }];
         let levels = Levels {
             levels,
-            l0: Some(l0),
+            l0,
             ..Default::default()
         };
 
@@ -626,7 +626,7 @@ pub mod tests {
         let l0 = generate_l0_nonoverlapping_sublevels(vec![]);
         let levels = vec![Level {
             level_idx: 1,
-            level_type: LevelType::Nonoverlapping as i32,
+            level_type: LevelType::Nonoverlapping,
             table_infos: vec![],
             total_file_size: 0,
             sub_level_id: 0,
@@ -635,7 +635,7 @@ pub mod tests {
         }];
         let levels = Levels {
             levels,
-            l0: Some(l0),
+            l0,
             ..Default::default()
         };
         let levels_handler = vec![LevelHandler::new(0), LevelHandler::new(1)];
@@ -643,8 +643,8 @@ pub mod tests {
             sst_ids: vec![1],
             level: 0,
             key_range: KeyRange {
-                left: vec![],
-                right: vec![],
+                left: Bytes::default(),
+                right: Bytes::default(),
                 right_exclusive: false,
             },
             internal_table_id: HashSet::default(),
@@ -669,8 +669,8 @@ pub mod tests {
             sst_ids: vec![],
             level: 0,
             key_range: KeyRange {
-                left: vec![],
-                right: vec![],
+                left: Bytes::default(),
+                right: Bytes::default(),
                 right_exclusive: false,
             },
             internal_table_id: HashSet::default(),
@@ -701,7 +701,7 @@ pub mod tests {
                 result.input_levels[l]
                     .table_infos
                     .iter()
-                    .map(|s| s.get_sst_id())
+                    .map(|s| s.sst_id)
                     .collect_vec(),
                 *e
             );
@@ -716,8 +716,8 @@ pub mod tests {
             sst_ids: vec![],
             level: 0,
             key_range: KeyRange {
-                left: iterator_test_key_of_epoch(1, 0, 2),
-                right: iterator_test_key_of_epoch(1, 200, 2),
+                left: Bytes::from(iterator_test_key_of_epoch(1, 0, 2)),
+                right: Bytes::from(iterator_test_key_of_epoch(1, 200, 2)),
                 right_exclusive: false,
             },
             internal_table_id: HashSet::default(),
@@ -737,7 +737,7 @@ pub mod tests {
                 result.input_levels[l]
                     .table_infos
                     .iter()
-                    .map(|s| s.get_sst_id())
+                    .map(|s| s.sst_id)
                     .collect_vec(),
                 *e
             );
@@ -767,8 +767,8 @@ pub mod tests {
                 sst_ids: sst_id_filter.clone(),
                 level: *input_level as _,
                 key_range: KeyRange {
-                    left: vec![],
-                    right: vec![],
+                    left: Bytes::default(),
+                    right: Bytes::default(),
                     right_exclusive: false,
                 },
                 internal_table_id: HashSet::default(),
@@ -789,7 +789,7 @@ pub mod tests {
                     result.input_levels[i]
                         .table_infos
                         .iter()
-                        .map(|s| s.get_sst_id())
+                        .map(|s| s.sst_id)
                         .collect_vec(),
                     *e
                 );
@@ -808,8 +808,8 @@ pub mod tests {
                 sst_ids: vec![],
                 level: input_level,
                 key_range: KeyRange {
-                    left: vec![],
-                    right: vec![],
+                    left: Bytes::default(),
+                    right: Bytes::default(),
                     right_exclusive: false,
                 },
                 // No matching internal table id.
@@ -830,8 +830,8 @@ pub mod tests {
                 sst_ids: vec![],
                 level: input_level,
                 key_range: KeyRange {
-                    left: vec![],
-                    right: vec![],
+                    left: Bytes::default(),
+                    right: Bytes::default(),
                     right_exclusive: false,
                 },
                 // Include all sub level's table ids
@@ -855,7 +855,7 @@ pub mod tests {
                     .iter()
                     .take(3)
                     .flat_map(|s| s.table_infos.clone())
-                    .map(|s| s.get_sst_id())
+                    .map(|s| s.sst_id)
                     .collect_vec(),
                 vec![9, 10, 7, 8, 5, 6]
             );
@@ -863,7 +863,7 @@ pub mod tests {
                 result.input_levels[3]
                     .table_infos
                     .iter()
-                    .map(|s| s.get_sst_id())
+                    .map(|s| s.sst_id)
                     .collect_vec(),
                 vec![3]
             );
@@ -874,8 +874,8 @@ pub mod tests {
                 sst_ids: vec![],
                 level: input_level,
                 key_range: KeyRange {
-                    left: vec![],
-                    right: vec![],
+                    left: Bytes::default(),
+                    right: Bytes::default(),
                     right_exclusive: false,
                 },
                 // Only include bottom sub level's table id
@@ -897,7 +897,7 @@ pub mod tests {
                     .iter()
                     .take(3)
                     .flat_map(|s| s.table_infos.clone())
-                    .map(|s| s.get_sst_id())
+                    .map(|s| s.sst_id)
                     .collect_vec(),
                 vec![9, 10, 7, 8, 5, 6]
             );
@@ -905,7 +905,7 @@ pub mod tests {
                 result.input_levels[3]
                     .table_infos
                     .iter()
-                    .map(|s| s.get_sst_id())
+                    .map(|s| s.sst_id)
                     .collect_vec(),
                 vec![3]
             );
@@ -917,8 +917,8 @@ pub mod tests {
                 sst_ids: vec![],
                 level: input_level,
                 key_range: KeyRange {
-                    left: vec![],
-                    right: vec![],
+                    left: Bytes::default(),
+                    right: Bytes::default(),
                     right_exclusive: false,
                 },
                 // Only include partial top sub level's table id, but the whole top sub level is
@@ -943,7 +943,7 @@ pub mod tests {
                     .iter()
                     .take(1)
                     .flat_map(|s| s.table_infos.clone())
-                    .map(|s| s.get_sst_id())
+                    .map(|s| s.sst_id)
                     .collect_vec(),
                 vec![5, 6]
             );
@@ -951,7 +951,7 @@ pub mod tests {
                 result.input_levels[1]
                     .table_infos
                     .iter()
-                    .map(|s| s.get_sst_id())
+                    .map(|s| s.sst_id)
                     .collect_vec(),
                 vec![3]
             );
@@ -961,8 +961,8 @@ pub mod tests {
                 sst_ids: vec![],
                 level: input_level,
                 key_range: KeyRange {
-                    left: vec![],
-                    right: vec![],
+                    left: Bytes::default(),
+                    right: Bytes::default(),
                     right_exclusive: false,
                 },
                 // Only include bottom sub level's table id
@@ -994,8 +994,8 @@ pub mod tests {
                 sst_ids: vec![],
                 level: input_level,
                 key_range: KeyRange {
-                    left: vec![],
-                    right: vec![],
+                    left: Bytes::default(),
+                    right: Bytes::default(),
                     right_exclusive: false,
                 },
                 // No matching internal table id.
@@ -1017,8 +1017,8 @@ pub mod tests {
                 sst_ids: vec![],
                 level: input_level,
                 key_range: KeyRange {
-                    left: vec![],
-                    right: vec![],
+                    left: Bytes::default(),
+                    right: Bytes::default(),
                     right_exclusive: false,
                 },
                 // Only include partial input level's table id
@@ -1046,7 +1046,7 @@ pub mod tests {
                     result.input_levels[l]
                         .table_infos
                         .iter()
-                        .map(|s| s.get_sst_id())
+                        .map(|s| s.sst_id)
                         .collect_vec(),
                     *e
                 );
@@ -1069,8 +1069,8 @@ pub mod tests {
                 sst_ids: sst_id_filter.clone(),
                 level: *input_level as _,
                 key_range: KeyRange {
-                    left: vec![],
-                    right: vec![],
+                    left: Bytes::default(),
+                    right: Bytes::default(),
                     right_exclusive: false,
                 },
                 internal_table_id: HashSet::default(),
@@ -1089,7 +1089,7 @@ pub mod tests {
                     result.input_levels[i]
                         .table_infos
                         .iter()
-                        .map(|s| s.get_sst_id())
+                        .map(|s| s.sst_id)
                         .collect_vec(),
                     *e
                 );
@@ -1115,8 +1115,8 @@ pub mod tests {
                 sst_ids: sst_id_filter.clone(),
                 level: *input_level as _,
                 key_range: KeyRange {
-                    left: vec![],
-                    right: vec![],
+                    left: Bytes::default(),
+                    right: Bytes::default(),
                     right_exclusive: false,
                 },
                 internal_table_id: HashSet::default(),
@@ -1139,7 +1139,7 @@ pub mod tests {
                     result.input_levels[i]
                         .table_infos
                         .iter()
-                        .map(|s| s.get_sst_id())
+                        .map(|s| s.sst_id)
                         .collect_vec(),
                     *e
                 );
@@ -1162,7 +1162,7 @@ pub mod tests {
             generate_level(3, vec![]),
             Level {
                 level_idx: 4,
-                level_type: LevelType::Nonoverlapping as i32,
+                level_type: LevelType::Nonoverlapping,
                 table_infos: vec![
                     generate_table(2, 1, 0, 100, 1),
                     generate_table(3, 1, 101, 200, 1),
@@ -1174,7 +1174,7 @@ pub mod tests {
         assert_eq!(levels.len(), 4);
         let levels = Levels {
             levels,
-            l0: Some(l0),
+            l0,
             ..Default::default()
         };
         let mut levels_handler = (0..5).map(LevelHandler::new).collect_vec();
@@ -1185,8 +1185,8 @@ pub mod tests {
             let option = ManualCompactionOption {
                 sst_ids: vec![0, 1],
                 key_range: KeyRange {
-                    left: vec![],
-                    right: vec![],
+                    left: Bytes::default(),
+                    right: Bytes::default(),
                     right_exclusive: false,
                 },
                 internal_table_id: HashSet::default(),
@@ -1196,13 +1196,17 @@ pub mod tests {
             let task = selector
                 .pick_compaction(
                     1,
-                    &group_config,
-                    &levels,
-                    &BTreeSet::new(),
-                    &mut levels_handler,
-                    &mut local_stats,
-                    HashMap::default(),
-                    Arc::new(CompactionDeveloperConfig::default()),
+                    compaction_selector_context(
+                        &group_config,
+                        &levels,
+                        &BTreeSet::new(),
+                        &mut levels_handler,
+                        &mut local_stats,
+                        &HashMap::default(),
+                        Arc::new(CompactionDeveloperConfig::default()),
+                        &Default::default(),
+                        &HummockVersionStateTableInfo::empty(),
+                    ),
                 )
                 .unwrap();
             assert_compaction_task(&task, &levels_handler);
@@ -1223,8 +1227,8 @@ pub mod tests {
             let option = ManualCompactionOption {
                 sst_ids: vec![],
                 key_range: KeyRange {
-                    left: vec![],
-                    right: vec![],
+                    left: Bytes::default(),
+                    right: Bytes::default(),
                     right_exclusive: false,
                 },
                 internal_table_id: HashSet::default(),
@@ -1234,13 +1238,17 @@ pub mod tests {
             let task = selector
                 .pick_compaction(
                     2,
-                    &group_config,
-                    &levels,
-                    &BTreeSet::new(),
-                    &mut levels_handler,
-                    &mut local_stats,
-                    HashMap::default(),
-                    Arc::new(CompactionDeveloperConfig::default()),
+                    compaction_selector_context(
+                        &group_config,
+                        &levels,
+                        &BTreeSet::new(),
+                        &mut levels_handler,
+                        &mut local_stats,
+                        &HashMap::default(),
+                        Arc::new(CompactionDeveloperConfig::default()),
+                        &Default::default(),
+                        &HummockVersionStateTableInfo::empty(),
+                    ),
                 )
                 .unwrap();
             assert_compaction_task(&task, &levels_handler);
@@ -1271,7 +1279,7 @@ pub mod tests {
             ),
             Level {
                 level_idx: 4,
-                level_type: LevelType::Nonoverlapping as i32,
+                level_type: LevelType::Nonoverlapping,
                 table_infos: vec![
                     generate_table(2, 1, 0, 100, 1),
                     generate_table(3, 1, 101, 200, 1),
@@ -1286,7 +1294,7 @@ pub mod tests {
         assert_eq!(levels.len(), 4);
         let levels = Levels {
             levels,
-            l0: Some(l0),
+            l0,
             ..Default::default()
         };
         let mut levels_handler = (0..5).map(LevelHandler::new).collect_vec();
@@ -1297,8 +1305,8 @@ pub mod tests {
             let option = ManualCompactionOption {
                 sst_ids: vec![0, 1],
                 key_range: KeyRange {
-                    left: vec![],
-                    right: vec![],
+                    left: Bytes::default(),
+                    right: Bytes::default(),
                     right_exclusive: false,
                 },
                 internal_table_id: HashSet::default(),
@@ -1308,13 +1316,17 @@ pub mod tests {
             let task = selector
                 .pick_compaction(
                     1,
-                    &group_config,
-                    &levels,
-                    &BTreeSet::new(),
-                    &mut levels_handler,
-                    &mut local_stats,
-                    HashMap::default(),
-                    Arc::new(CompactionDeveloperConfig::default()),
+                    compaction_selector_context(
+                        &group_config,
+                        &levels,
+                        &BTreeSet::new(),
+                        &mut levels_handler,
+                        &mut local_stats,
+                        &HashMap::default(),
+                        Arc::new(CompactionDeveloperConfig::default()),
+                        &Default::default(),
+                        &HummockVersionStateTableInfo::empty(),
+                    ),
                 )
                 .unwrap();
             assert_compaction_task(&task, &levels_handler);
@@ -1337,8 +1349,8 @@ pub mod tests {
             let option = ManualCompactionOption {
                 sst_ids: vec![],
                 key_range: KeyRange {
-                    left: vec![],
-                    right: vec![],
+                    left: Bytes::default(),
+                    right: Bytes::default(),
                     right_exclusive: false,
                 },
                 internal_table_id: HashSet::default(),
@@ -1348,13 +1360,17 @@ pub mod tests {
             let task = selector
                 .pick_compaction(
                     1,
-                    &group_config,
-                    &levels,
-                    &BTreeSet::new(),
-                    &mut levels_handler,
-                    &mut local_stats,
-                    HashMap::default(),
-                    Arc::new(CompactionDeveloperConfig::default()),
+                    compaction_selector_context(
+                        &group_config,
+                        &levels,
+                        &BTreeSet::new(),
+                        &mut levels_handler,
+                        &mut local_stats,
+                        &HashMap::default(),
+                        Arc::new(CompactionDeveloperConfig::default()),
+                        &Default::default(),
+                        &HummockVersionStateTableInfo::empty(),
+                    ),
                 )
                 .unwrap();
             assert_compaction_task(&task, &levels_handler);
diff --git a/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs
index 57dd5469d42ae..dfbe6950c5b49 100644
--- a/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs
+++ b/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs
@@ -15,10 +15,9 @@
 use std::sync::Arc;
 
 use risingwave_hummock_sdk::append_sstable_info_to_string;
-use risingwave_hummock_sdk::compaction_group::hummock_version_ext::HummockLevelsExt;
-use risingwave_hummock_sdk::prost_key_range::KeyRangeExt;
-use risingwave_pb::hummock::hummock_version::Levels;
-use risingwave_pb::hummock::{InputLevel, Level, LevelType, SstableInfo};
+use risingwave_hummock_sdk::level::{InputLevel, Level, Levels};
+use risingwave_hummock_sdk::sstable_info::SstableInfo;
+use risingwave_pb::hummock::LevelType;
 
 use super::{CompactionInput, CompactionPicker, LocalPickerStatistic};
 use crate::hummock::compaction::overlap_strategy::OverlapStrategy;
@@ -156,12 +155,12 @@ impl CompactionPicker for MinOverlappingPicker {
             input_levels: vec![
                 InputLevel {
                     level_idx: self.level as u32,
-                    level_type: LevelType::Nonoverlapping as i32,
+                    level_type: LevelType::Nonoverlapping,
                     table_infos: select_input_ssts,
                 },
                 InputLevel {
                     level_idx: self.target_level as u32,
-                    level_type: LevelType::Nonoverlapping as i32,
+                    level_type: LevelType::Nonoverlapping,
                     table_infos: target_input_ssts,
                 },
             ],
@@ -253,7 +252,7 @@ impl NonOverlapSubLevelPicker {
         // Pay attention to the order here: Make sure to select the lowest sub_level to meet the requirements of base compaction. If you break the assumption of this order, you need to redesign it.
         // TODO: Use binary selection to replace the step algorithm to optimize algorithm complexity
         'expand_new_level: for (target_index, target_level) in levels.iter().enumerate().skip(1) {
-            if target_level.level_type() != LevelType::Nonoverlapping {
+            if target_level.level_type != LevelType::Nonoverlapping {
                 break;
             }
 
@@ -310,7 +309,7 @@ impl NonOverlapSubLevelPicker {
                     }
                     basic_overlap_info.update(other);
 
-                    add_files_size += other.get_file_size();
+                    add_files_size += other.file_size;
                     add_files_count += 1;
                 }
 
@@ -345,11 +344,7 @@ impl NonOverlapSubLevelPicker {
 
             // sort sst per level due to reverse expand
             ret.sstable_infos.iter_mut().for_each(|level_ssts| {
-                level_ssts.sort_by(|sst1, sst2| {
-                    let a = sst1.key_range.as_ref().unwrap();
-                    let b = sst2.key_range.as_ref().unwrap();
-                    a.compare(b)
-                });
+                level_ssts.sort_by(|sst1, sst2| sst1.key_range.cmp(&sst2.key_range));
             });
         } else {
             ret.total_file_count = 1;
@@ -549,7 +544,7 @@ pub mod tests {
         let levels = vec![
             Level {
                 level_idx: 1,
-                level_type: LevelType::Nonoverlapping as i32,
+                level_type: LevelType::Nonoverlapping,
                 table_infos: vec![
                     generate_table(0, 1, 0, 100, 1),
                     generate_table(1, 1, 101, 200, 1),
@@ -559,7 +554,7 @@ pub mod tests {
             },
             Level {
                 level_idx: 2,
-                level_type: LevelType::Nonoverlapping as i32,
+                level_type: LevelType::Nonoverlapping,
                 table_infos: vec![
                     generate_table(4, 1, 0, 100, 1),
                     generate_table(5, 1, 101, 150, 1),
@@ -572,7 +567,7 @@ pub mod tests {
         ];
         let levels = Levels {
             levels,
-            l0: Some(generate_l0_nonoverlapping_sublevels(vec![])),
+            l0: generate_l0_nonoverlapping_sublevels(vec![]),
             ..Default::default()
         };
         let mut level_handlers = vec![
@@ -590,7 +585,7 @@ pub mod tests {
         assert_eq!(ret.input_levels[0].level_idx, 1);
         assert_eq!(ret.target_level, 2);
         assert_eq!(ret.input_levels[0].table_infos.len(), 1);
-        assert_eq!(ret.input_levels[0].table_infos[0].get_sst_id(), 2);
+        assert_eq!(ret.input_levels[0].table_infos[0].sst_id, 2);
         assert_eq!(ret.input_levels[1].table_infos.len(), 0);
         ret.add_pending_task(0, &mut level_handlers);
 
@@ -600,18 +595,18 @@ pub mod tests {
         assert_eq!(ret.input_levels[0].level_idx, 1);
         assert_eq!(ret.target_level, 2);
         assert_eq!(ret.input_levels[0].table_infos.len(), 1);
-        assert_eq!(ret.input_levels[0].table_infos[0].get_sst_id(), 0);
+        assert_eq!(ret.input_levels[0].table_infos[0].sst_id, 0);
         assert_eq!(ret.input_levels[1].table_infos.len(), 1);
-        assert_eq!(ret.input_levels[1].table_infos[0].get_sst_id(), 4);
+        assert_eq!(ret.input_levels[1].table_infos[0].sst_id, 4);
         ret.add_pending_task(1, &mut level_handlers);
 
         let ret = picker
             .pick_compaction(&levels, &level_handlers, &mut local_stats)
             .unwrap();
         assert_eq!(ret.input_levels[0].table_infos.len(), 1);
-        assert_eq!(ret.input_levels[0].table_infos[0].get_sst_id(), 1);
+        assert_eq!(ret.input_levels[0].table_infos[0].sst_id, 1);
         assert_eq!(ret.input_levels[1].table_infos.len(), 2);
-        assert_eq!(ret.input_levels[1].table_infos[0].get_sst_id(), 5);
+        assert_eq!(ret.input_levels[1].table_infos[0].sst_id, 5);
     }
 
     #[test]
@@ -621,7 +616,7 @@ pub mod tests {
         let levels = vec![
             Level {
                 level_idx: 1,
-                level_type: LevelType::Nonoverlapping as i32,
+                level_type: LevelType::Nonoverlapping,
                 table_infos: vec![
                     generate_table(0, 1, 50, 99, 2),
                     generate_table(1, 1, 100, 149, 2),
@@ -631,7 +626,7 @@ pub mod tests {
             },
             Level {
                 level_idx: 2,
-                level_type: LevelType::Nonoverlapping as i32,
+                level_type: LevelType::Nonoverlapping,
                 table_infos: vec![
                     generate_table(4, 1, 50, 199, 1),
                     generate_table(5, 1, 200, 399, 1),
@@ -641,7 +636,7 @@ pub mod tests {
         ];
         let levels = Levels {
             levels,
-            l0: Some(generate_l0_nonoverlapping_sublevels(vec![])),
+            l0: generate_l0_nonoverlapping_sublevels(vec![]),
             ..Default::default()
         };
         let levels_handler = vec![
@@ -663,11 +658,11 @@ pub mod tests {
         assert_eq!(ret.input_levels[1].level_idx, 2);
 
         assert_eq!(ret.input_levels[0].table_infos.len(), 2);
-        assert_eq!(ret.input_levels[0].table_infos[0].get_sst_id(), 0);
-        assert_eq!(ret.input_levels[0].table_infos[1].get_sst_id(), 1);
+        assert_eq!(ret.input_levels[0].table_infos[0].sst_id, 0);
+        assert_eq!(ret.input_levels[0].table_infos[1].sst_id, 1);
 
         assert_eq!(ret.input_levels[1].table_infos.len(), 1);
-        assert_eq!(ret.input_levels[1].table_infos[0].get_sst_id(), 4);
+        assert_eq!(ret.input_levels[1].table_infos[0].sst_id, 4);
     }
 
     #[test]
@@ -675,7 +670,7 @@ pub mod tests {
         let levels = vec![
             Level {
                 level_idx: 1,
-                level_type: LevelType::Nonoverlapping as i32,
+                level_type: LevelType::Nonoverlapping,
                 table_infos: vec![
                     generate_table(0, 1, 50, 99, 2),
                     generate_table(1, 1, 100, 149, 2),
@@ -689,7 +684,7 @@ pub mod tests {
             },
             Level {
                 level_idx: 2,
-                level_type: LevelType::Nonoverlapping as i32,
+                level_type: LevelType::Nonoverlapping,
                 table_infos: vec![
                     generate_table(4, 1, 50, 199, 1),
                     generate_table(5, 1, 200, 249, 1),
@@ -702,7 +697,7 @@ pub mod tests {
             },
             Level {
                 level_idx: 3,
-                level_type: LevelType::Nonoverlapping as i32,
+                level_type: LevelType::Nonoverlapping,
                 table_infos: vec![
                     generate_table(11, 1, 250, 300, 2),
                     generate_table(12, 1, 350, 400, 2),
@@ -713,7 +708,7 @@ pub mod tests {
             },
             Level {
                 level_idx: 4,
-                level_type: LevelType::Nonoverlapping as i32,
+                level_type: LevelType::Nonoverlapping,
                 table_infos: vec![
                     generate_table(14, 1, 250, 300, 2),
                     generate_table(15, 1, 350, 400, 2),
@@ -781,7 +776,7 @@ pub mod tests {
         let levels = vec![
             Level {
                 level_idx: 1,
-                level_type: LevelType::Nonoverlapping as i32,
+                level_type: LevelType::Nonoverlapping,
                 table_infos: vec![
                     generate_table(0, 1, 50, 99, 2),
                     generate_table(1, 1, 100, 149, 2),
@@ -795,7 +790,7 @@ pub mod tests {
             },
             Level {
                 level_idx: 2,
-                level_type: LevelType::Nonoverlapping as i32,
+                level_type: LevelType::Nonoverlapping,
                 table_infos: vec![
                     generate_table(4, 1, 50, 99, 1),
                     generate_table(5, 1, 150, 200, 1),
@@ -808,7 +803,7 @@ pub mod tests {
             },
             Level {
                 level_idx: 3,
-                level_type: LevelType::Nonoverlapping as i32,
+                level_type: LevelType::Nonoverlapping,
                 table_infos: vec![
                     generate_table(11, 1, 250, 300, 2),
                     generate_table(12, 1, 350, 400, 2),
@@ -819,7 +814,7 @@ pub mod tests {
             },
             Level {
                 level_idx: 4,
-                level_type: LevelType::Nonoverlapping as i32,
+                level_type: LevelType::Nonoverlapping,
                 table_infos: vec![
                     generate_table(14, 1, 250, 300, 2),
                     generate_table(15, 1, 350, 400, 2),
@@ -885,7 +880,7 @@ pub mod tests {
             for plan in ret {
                 let mut sst_id_set = BTreeSet::default();
                 for sst in &plan.sstable_infos {
-                    sst_id_set.insert(sst[0].get_sst_id());
+                    sst_id_set.insert(sst[0].sst_id);
                 }
                 assert!(sst_id_set.len() <= max_file_count as usize);
             }
@@ -909,7 +904,7 @@ pub mod tests {
             for plan in ret {
                 let mut sst_id_set = BTreeSet::default();
                 for sst in &plan.sstable_infos {
-                    sst_id_set.insert(sst[0].get_sst_id());
+                    sst_id_set.insert(sst[0].sst_id);
                 }
                 assert!(plan.sstable_infos.len() >= min_depth);
             }
@@ -921,14 +916,14 @@ pub mod tests {
         let levels = [
             Level {
                 level_idx: 1,
-                level_type: LevelType::Nonoverlapping as i32,
+                level_type: LevelType::Nonoverlapping,
                 table_infos: vec![generate_table(0, 1, 400, 500, 2)],
                 total_file_size: 100,
                 ..Default::default()
             },
             Level {
                 level_idx: 2,
-                level_type: LevelType::Nonoverlapping as i32,
+                level_type: LevelType::Nonoverlapping,
                 table_infos: vec![
                     generate_table(1, 1, 100, 200, 1),
                     generate_table(2, 1, 600, 700, 1),
@@ -938,7 +933,7 @@ pub mod tests {
             },
             Level {
                 level_idx: 3,
-                level_type: LevelType::Nonoverlapping as i32,
+                level_type: LevelType::Nonoverlapping,
                 table_infos: vec![
                     generate_table(3, 1, 100, 300, 2),
                     generate_table(4, 1, 600, 800, 1),
@@ -978,14 +973,14 @@ pub mod tests {
         let levels = vec![
             Level {
                 level_idx: 1,
-                level_type: LevelType::Nonoverlapping as i32,
+                level_type: LevelType::Nonoverlapping,
                 table_infos: vec![generate_table(0, 1, 50, 100, 2)], // 50
                 total_file_size: 50,
                 ..Default::default()
             },
             Level {
                 level_idx: 2,
-                level_type: LevelType::Nonoverlapping as i32,
+                level_type: LevelType::Nonoverlapping,
                 table_infos: vec![
                     generate_table(1, 1, 101, 150, 1), // 50
                 ],
@@ -994,7 +989,7 @@ pub mod tests {
             },
             Level {
                 level_idx: 3,
-                level_type: LevelType::Nonoverlapping as i32,
+                level_type: LevelType::Nonoverlapping,
                 table_infos: vec![
                     generate_table(2, 1, 151, 200, 2), // 50
                 ],
@@ -1003,7 +998,7 @@ pub mod tests {
             },
             Level {
                 level_idx: 4,
-                level_type: LevelType::Nonoverlapping as i32,
+                level_type: LevelType::Nonoverlapping,
                 table_infos: vec![
                     generate_table(3, 1, 50, 300, 2), // 250
                 ],
diff --git a/src/meta/src/hummock/compaction/picker/mod.rs b/src/meta/src/hummock/compaction/picker/mod.rs
index 6d464b9a33bcd..3bed84fc57b98 100644
--- a/src/meta/src/hummock/compaction/picker/mod.rs
+++ b/src/meta/src/hummock/compaction/picker/mod.rs
@@ -24,6 +24,7 @@ mod trivial_move_compaction_picker;
 mod ttl_reclaim_compaction_picker;
 
 mod compaction_task_validator;
+mod vnode_watermark_picker;
 
 pub use base_level_compaction_picker::LevelCompactionPicker;
 pub use compaction_task_validator::{CompactionTaskValidator, ValidationRuleType};
@@ -31,8 +32,7 @@ pub use emergency_compaction_picker::EmergencyCompactionPicker;
 pub use intra_compaction_picker::IntraCompactionPicker;
 pub use manual_compaction_picker::ManualCompactionPicker;
 pub use min_overlap_compaction_picker::MinOverlappingPicker;
-use risingwave_pb::hummock::hummock_version::Levels;
-use risingwave_pb::hummock::InputLevel;
+use risingwave_hummock_sdk::level::{InputLevel, Levels};
 pub use space_reclaim_compaction_picker::{SpaceReclaimCompactionPicker, SpaceReclaimPickerState};
 pub use tier_compaction_picker::TierCompactionPicker;
 pub use tombstone_reclaim_compaction_picker::{
@@ -40,6 +40,7 @@ pub use tombstone_reclaim_compaction_picker::{
 };
 pub use trivial_move_compaction_picker::TrivialMovePicker;
 pub use ttl_reclaim_compaction_picker::{TtlPickerState, TtlReclaimCompactionPicker};
+pub use vnode_watermark_picker::VnodeWatermarkCompactionPicker;
 
 use crate::hummock::level_handler::LevelHandler;
 
diff --git a/src/meta/src/hummock/compaction/picker/space_reclaim_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/space_reclaim_compaction_picker.rs
index b94f7587dd04e..7a4eb86831c33 100644
--- a/src/meta/src/hummock/compaction/picker/space_reclaim_compaction_picker.rs
+++ b/src/meta/src/hummock/compaction/picker/space_reclaim_compaction_picker.rs
@@ -14,8 +14,8 @@
 
 use std::collections::HashSet;
 
-use risingwave_pb::hummock::hummock_version::Levels;
-use risingwave_pb::hummock::{InputLevel, SstableInfo};
+use risingwave_hummock_sdk::level::{InputLevel, Levels};
+use risingwave_hummock_sdk::sstable_info::SstableInfo;
 
 use super::CompactionInput;
 use crate::hummock::level_handler::LevelHandler;
@@ -63,9 +63,9 @@ impl SpaceReclaimCompactionPicker {
     ) -> Option<CompactionInput> {
         assert!(!levels.levels.is_empty());
         let mut select_input_ssts = vec![];
-        if let Some(l0) = levels.l0.as_ref()
-            && state.last_level == 0
-        {
+
+        if state.last_level == 0 {
+            let l0 = &levels.l0;
             // only pick trivial reclaim sstables because this kind of task could be optimized and do not need send to compactor.
             for level in &l0.sub_levels {
                 for sst in &level.table_infos {
@@ -173,8 +173,10 @@ mod test {
 
     use itertools::Itertools;
     use risingwave_common::catalog::TableId;
+    use risingwave_hummock_sdk::level::Level;
+    use risingwave_hummock_sdk::version::HummockVersionStateTableInfo;
     use risingwave_pb::hummock::compact_task;
-    pub use risingwave_pb::hummock::{Level, LevelType};
+    pub use risingwave_pb::hummock::LevelType;
 
     use super::*;
     use crate::hummock::compaction::compaction_config::CompactionConfigBuilder;
@@ -187,6 +189,7 @@ mod test {
     };
     use crate::hummock::compaction::CompactionDeveloperConfig;
     use crate::hummock::model::CompactionGroup;
+    use crate::hummock::test_utils::compaction_selector_context;
 
     #[test]
     fn test_space_reclaim_compaction_selector() {
@@ -211,7 +214,7 @@ mod test {
             ),
             Level {
                 level_idx: 4,
-                level_type: LevelType::Nonoverlapping as i32,
+                level_type: LevelType::Nonoverlapping,
                 table_infos: vec![
                     generate_table_with_ids_and_epochs(2, 1, 0, 100, 1, vec![2], 0, 0),
                     generate_table_with_ids_and_epochs(3, 1, 101, 200, 1, vec![3], 0, 0),
@@ -230,14 +233,14 @@ mod test {
 
         {
             let sst_10 = levels[3].table_infos.get_mut(8).unwrap();
-            assert_eq!(10, sst_10.get_sst_id());
-            sst_10.key_range.as_mut().unwrap().right_exclusive = true;
+            assert_eq!(10, sst_10.sst_id);
+            sst_10.key_range.right_exclusive = true;
         }
 
         assert_eq!(levels.len(), 4);
         let levels = Levels {
             levels,
-            l0: Some(l0),
+            l0,
             ..Default::default()
         };
         let mut member_table_ids = BTreeSet::new();
@@ -252,13 +255,17 @@ mod test {
             let task = selector
                 .pick_compaction(
                     1,
-                    &group_config,
-                    &levels,
-                    &member_table_ids,
-                    &mut levels_handler,
-                    &mut local_stats,
-                    HashMap::default(),
-                    Arc::new(CompactionDeveloperConfig::default()),
+                    compaction_selector_context(
+                        &group_config,
+                        &levels,
+                        &member_table_ids,
+                        &mut levels_handler,
+                        &mut local_stats,
+                        &HashMap::default(),
+                        Arc::new(CompactionDeveloperConfig::default()),
+                        &Default::default(),
+                        &HummockVersionStateTableInfo::empty(),
+                    ),
                 )
                 .unwrap();
             assert_compaction_task(&task, &levels_handler);
@@ -269,13 +276,17 @@ mod test {
             let task = selector
                 .pick_compaction(
                     1,
-                    &group_config,
-                    &levels,
-                    &member_table_ids,
-                    &mut levels_handler,
-                    &mut local_stats,
-                    HashMap::default(),
-                    Arc::new(CompactionDeveloperConfig::default()),
+                    compaction_selector_context(
+                        &group_config,
+                        &levels,
+                        &member_table_ids,
+                        &mut levels_handler,
+                        &mut local_stats,
+                        &HashMap::default(),
+                        Arc::new(CompactionDeveloperConfig::default()),
+                        &Default::default(),
+                        &HummockVersionStateTableInfo::empty(),
+                    ),
                 )
                 .unwrap();
             assert_eq!(task.input.input_levels.len(), 2);
@@ -284,7 +295,7 @@ mod test {
 
             let mut start_id = 2;
             for sst in &task.input.input_levels[0].table_infos {
-                assert_eq!(start_id, sst.get_sst_id());
+                assert_eq!(start_id, sst.sst_id);
                 start_id += 1;
             }
 
@@ -310,13 +321,17 @@ mod test {
             let task = selector
                 .pick_compaction(
                     1,
-                    &group_config,
-                    &levels,
-                    &member_table_ids,
-                    &mut levels_handler,
-                    &mut local_stats,
-                    HashMap::default(),
-                    Arc::new(CompactionDeveloperConfig::default()),
+                    compaction_selector_context(
+                        &group_config,
+                        &levels,
+                        &member_table_ids,
+                        &mut levels_handler,
+                        &mut local_stats,
+                        &HashMap::default(),
+                        Arc::new(CompactionDeveloperConfig::default()),
+                        &Default::default(),
+                        &HummockVersionStateTableInfo::empty(),
+                    ),
                 )
                 .unwrap();
             assert_compaction_task(&task, &levels_handler);
@@ -330,20 +345,24 @@ mod test {
             ));
             let mut start_id = 8;
             for sst in &task.input.input_levels[0].table_infos {
-                assert_eq!(start_id, sst.get_sst_id());
+                assert_eq!(start_id, sst.sst_id);
                 start_id += 1;
             }
 
             assert!(selector
                 .pick_compaction(
                     1,
-                    &group_config,
-                    &levels,
-                    &member_table_ids,
-                    &mut levels_handler,
-                    &mut local_stats,
-                    HashMap::default(),
-                    Arc::new(CompactionDeveloperConfig::default()),
+                    compaction_selector_context(
+                        &group_config,
+                        &levels,
+                        &member_table_ids,
+                        &mut levels_handler,
+                        &mut local_stats,
+                        &HashMap::default(),
+                        Arc::new(CompactionDeveloperConfig::default()),
+                        &Default::default(),
+                        &HummockVersionStateTableInfo::empty(),
+                    ),
                 )
                 .is_none())
         }
@@ -365,13 +384,17 @@ mod test {
             // pick space reclaim
             let task = selector.pick_compaction(
                 1,
-                &group_config,
-                &levels,
-                &member_table_ids,
-                &mut levels_handler,
-                &mut local_stats,
-                HashMap::default(),
-                Arc::new(CompactionDeveloperConfig::default()),
+                compaction_selector_context(
+                    &group_config,
+                    &levels,
+                    &member_table_ids,
+                    &mut levels_handler,
+                    &mut local_stats,
+                    &HashMap::default(),
+                    Arc::new(CompactionDeveloperConfig::default()),
+                    &Default::default(),
+                    &HummockVersionStateTableInfo::empty(),
+                ),
             );
             assert!(task.is_none());
         }
@@ -389,13 +412,17 @@ mod test {
             let task = selector
                 .pick_compaction(
                     1,
-                    &group_config,
-                    &levels,
-                    &member_table_ids,
-                    &mut levels_handler,
-                    &mut local_stats,
-                    HashMap::default(),
-                    Arc::new(CompactionDeveloperConfig::default()),
+                    compaction_selector_context(
+                        &group_config,
+                        &levels,
+                        &member_table_ids,
+                        &mut levels_handler,
+                        &mut local_stats,
+                        &HashMap::default(),
+                        Arc::new(CompactionDeveloperConfig::default()),
+                        &Default::default(),
+                        &HummockVersionStateTableInfo::empty(),
+                    ),
                 )
                 .unwrap();
             assert_compaction_task(&task, &levels_handler);
@@ -428,13 +455,17 @@ mod test {
                 let task = selector
                     .pick_compaction(
                         1,
-                        &group_config,
-                        &levels,
-                        &member_table_ids,
-                        &mut levels_handler,
-                        &mut local_stats,
-                        HashMap::default(),
-                        Arc::new(CompactionDeveloperConfig::default()),
+                        compaction_selector_context(
+                            &group_config,
+                            &levels,
+                            &member_table_ids,
+                            &mut levels_handler,
+                            &mut local_stats,
+                            &HashMap::default(),
+                            Arc::new(CompactionDeveloperConfig::default()),
+                            &Default::default(),
+                            &HummockVersionStateTableInfo::empty(),
+                        ),
                     )
                     .unwrap();
 
@@ -446,7 +477,7 @@ mod test {
                 let select_sst = &task.input.input_levels[0]
                     .table_infos
                     .iter()
-                    .map(|sst| sst.get_sst_id())
+                    .map(|sst| sst.sst_id)
                     .collect_vec();
                 assert!(select_sst.is_sorted());
                 assert_eq!(expect_task_sst_id_range[index], *select_sst);
@@ -485,13 +516,17 @@ mod test {
                 let task = selector
                     .pick_compaction(
                         1,
-                        &group_config,
-                        &levels,
-                        &member_table_ids,
-                        &mut levels_handler,
-                        &mut local_stats,
-                        HashMap::default(),
-                        Arc::new(CompactionDeveloperConfig::default()),
+                        compaction_selector_context(
+                            &group_config,
+                            &levels,
+                            &member_table_ids,
+                            &mut levels_handler,
+                            &mut local_stats,
+                            &HashMap::default(),
+                            Arc::new(CompactionDeveloperConfig::default()),
+                            &Default::default(),
+                            &HummockVersionStateTableInfo::empty(),
+                        ),
                     )
                     .unwrap();
 
@@ -503,7 +538,7 @@ mod test {
                 let select_sst = &task.input.input_levels[0]
                     .table_infos
                     .iter()
-                    .map(|sst| sst.get_sst_id())
+                    .map(|sst| sst.sst_id)
                     .collect_vec();
                 assert!(select_sst.is_sorted());
                 assert_eq!(expect_task_sst_id_range[index], *select_sst);
diff --git a/src/meta/src/hummock/compaction/picker/tier_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/tier_compaction_picker.rs
index ea25211b44749..9099550232e3f 100644
--- a/src/meta/src/hummock/compaction/picker/tier_compaction_picker.rs
+++ b/src/meta/src/hummock/compaction/picker/tier_compaction_picker.rs
@@ -14,8 +14,8 @@
 
 use std::sync::Arc;
 
-use risingwave_pb::hummock::hummock_version::Levels;
-use risingwave_pb::hummock::{CompactionConfig, InputLevel, LevelType, OverlappingLevel};
+use risingwave_hummock_sdk::level::{InputLevel, Levels, OverlappingLevel};
+use risingwave_pb::hummock::{CompactionConfig, LevelType};
 
 use super::{
     CompactionInput, CompactionPicker, CompactionTaskValidator, LocalPickerStatistic,
@@ -55,7 +55,7 @@ impl TierCompactionPicker {
         stats: &mut LocalPickerStatistic,
     ) -> Option<CompactionInput> {
         for (idx, level) in l0.sub_levels.iter().enumerate() {
-            if level.level_type() != LevelType::Overlapping {
+            if level.level_type != LevelType::Overlapping {
                 continue;
             }
 
@@ -146,7 +146,7 @@ impl CompactionPicker for TierCompactionPicker {
         level_handlers: &[LevelHandler],
         stats: &mut LocalPickerStatistic,
     ) -> Option<CompactionInput> {
-        let l0 = levels.l0.as_ref().unwrap();
+        let l0 = &levels.l0;
         if l0.sub_levels.is_empty() {
             return None;
         }
@@ -165,8 +165,8 @@ pub mod tests {
     use std::sync::Arc;
 
     use risingwave_hummock_sdk::compaction_group::hummock_version_ext::new_sub_level;
-    use risingwave_pb::hummock::hummock_version::Levels;
-    use risingwave_pb::hummock::{LevelType, OverlappingLevel};
+    use risingwave_hummock_sdk::level::{Levels, OverlappingLevel};
+    use risingwave_pb::hummock::LevelType;
 
     use crate::hummock::compaction::compaction_config::CompactionConfigBuilder;
     use crate::hummock::compaction::picker::{
@@ -195,7 +195,7 @@ pub mod tests {
             ],
         ]);
         let levels = Levels {
-            l0: Some(l0),
+            l0,
             levels: vec![],
             ..Default::default()
         };
@@ -222,7 +222,7 @@ pub mod tests {
         );
 
         let empty_level = Levels {
-            l0: Some(generate_l0_overlapping_sublevels(vec![])),
+            l0: generate_l0_overlapping_sublevels(vec![]),
             levels: vec![],
             ..Default::default()
         };
@@ -243,7 +243,7 @@ pub mod tests {
         ]);
 
         let levels = Levels {
-            l0: Some(l0),
+            l0,
             levels: vec![],
             ..Default::default()
         };
@@ -285,11 +285,11 @@ pub mod tests {
             ],
         );
         let levels = Levels {
-            l0: Some(OverlappingLevel {
+            l0: OverlappingLevel {
                 total_file_size: l1.total_file_size + l2.total_file_size,
                 uncompressed_file_size: l1.total_file_size + l2.total_file_size,
                 sub_levels: vec![l1, l2],
-            }),
+            },
             levels: vec![],
             ..Default::default()
         };
@@ -319,7 +319,7 @@ pub mod tests {
             generate_table(10, 1, 1, 100, 1),
         ]]);
         let mut levels = Levels {
-            l0: Some(l0),
+            l0,
             levels: vec![],
             ..Default::default()
         };
diff --git a/src/meta/src/hummock/compaction/picker/tombstone_reclaim_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/tombstone_reclaim_compaction_picker.rs
index c7be0347fce61..ab6bfd1ed8490 100644
--- a/src/meta/src/hummock/compaction/picker/tombstone_reclaim_compaction_picker.rs
+++ b/src/meta/src/hummock/compaction/picker/tombstone_reclaim_compaction_picker.rs
@@ -14,8 +14,7 @@
 
 use std::sync::Arc;
 
-use risingwave_pb::hummock::hummock_version::Levels;
-use risingwave_pb::hummock::InputLevel;
+use risingwave_hummock_sdk::level::{InputLevel, Levels};
 
 use crate::hummock::compaction::overlap_strategy::OverlapStrategy;
 use crate::hummock::compaction::picker::CompactionInput;
@@ -133,8 +132,6 @@ impl TombstoneReclaimCompactionPicker {
 
 #[cfg(test)]
 pub mod tests {
-    use risingwave_pb::hummock::OverlappingLevel;
-
     use super::*;
     use crate::hummock::compaction::compaction_config::CompactionConfigBuilder;
     use crate::hummock::compaction::create_overlap_strategy;
@@ -143,7 +140,6 @@ pub mod tests {
     #[test]
     fn test_basic() {
         let mut levels = Levels {
-            l0: Some(OverlappingLevel::default()),
             levels: vec![
                 generate_level(1, vec![]),
                 generate_level(
diff --git a/src/meta/src/hummock/compaction/picker/trivial_move_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/trivial_move_compaction_picker.rs
index bab0dca0c3244..96fcc69175bdc 100644
--- a/src/meta/src/hummock/compaction/picker/trivial_move_compaction_picker.rs
+++ b/src/meta/src/hummock/compaction/picker/trivial_move_compaction_picker.rs
@@ -14,7 +14,9 @@
 
 use std::sync::Arc;
 
-use risingwave_pb::hummock::{InputLevel, LevelType, SstableInfo};
+use risingwave_hummock_sdk::level::InputLevel;
+use risingwave_hummock_sdk::sstable_info::SstableInfo;
+use risingwave_pb::hummock::LevelType;
 
 use super::{CompactionInput, LocalPickerStatistic};
 use crate::hummock::compaction::overlap_strategy::OverlapStrategy;
@@ -84,12 +86,12 @@ impl TrivialMovePicker {
                 input_levels: vec![
                     InputLevel {
                         level_idx: self.level as u32,
-                        level_type: LevelType::Nonoverlapping as i32,
+                        level_type: LevelType::Nonoverlapping,
                         table_infos: vec![trivial_move_sst],
                     },
                     InputLevel {
                         level_idx: self.target_level as u32,
-                        level_type: LevelType::Nonoverlapping as i32,
+                        level_type: LevelType::Nonoverlapping,
                         table_infos: vec![],
                     },
                 ],
diff --git a/src/meta/src/hummock/compaction/picker/ttl_reclaim_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/ttl_reclaim_compaction_picker.rs
index bc1fc2ce304be..5d3f17e31b6bc 100644
--- a/src/meta/src/hummock/compaction/picker/ttl_reclaim_compaction_picker.rs
+++ b/src/meta/src/hummock/compaction/picker/ttl_reclaim_compaction_picker.rs
@@ -14,12 +14,13 @@
 
 use std::collections::{HashMap, HashSet};
 
+use bytes::Bytes;
 use risingwave_common::catalog::TableOption;
 use risingwave_common::util::epoch::Epoch;
 use risingwave_hummock_sdk::compaction_group::StateTableId;
-use risingwave_hummock_sdk::key_range::KeyRangeCommon;
-use risingwave_pb::hummock::hummock_version::Levels;
-use risingwave_pb::hummock::{InputLevel, KeyRange, SstableInfo};
+use risingwave_hummock_sdk::key_range::{KeyRange, KeyRangeCommon};
+use risingwave_hummock_sdk::level::{InputLevel, Levels};
+use risingwave_hummock_sdk::sstable_info::SstableInfo;
 
 use super::CompactionInput;
 use crate::hummock::level_handler::LevelHandler;
@@ -46,7 +47,7 @@ impl TtlPickerState {
 
     pub fn init(&mut self, key_range: KeyRange) {
         self.last_select_end_bound = KeyRange {
-            left: vec![],
+            left: Bytes::default(),
             right: key_range.left.clone(),
             right_exclusive: true,
         };
@@ -64,7 +65,7 @@ pub struct TtlReclaimCompactionPicker {
 }
 
 impl TtlReclaimCompactionPicker {
-    pub fn new(table_id_to_options: HashMap<StateTableId, TableOption>) -> Self {
+    pub fn new(table_id_to_options: &HashMap<StateTableId, TableOption>) -> Self {
         let table_id_to_ttl: HashMap<u32, u32> = table_id_to_options
             .iter()
             .filter(|id_to_option| {
@@ -137,9 +138,9 @@ impl TtlReclaimCompactionPicker {
             let last_sst = reclaimed_level.table_infos.last().unwrap();
 
             let key_range_this_round = KeyRange {
-                left: first_sst.key_range.as_ref().unwrap().left.clone(),
-                right: last_sst.key_range.as_ref().unwrap().right.clone(),
-                right_exclusive: last_sst.key_range.as_ref().unwrap().right_exclusive,
+                left: first_sst.key_range.left.clone(),
+                right: last_sst.key_range.right.clone(),
+                right_exclusive: last_sst.key_range.right_exclusive,
             };
 
             state.init(key_range_this_round);
@@ -148,11 +149,7 @@ impl TtlReclaimCompactionPicker {
         let current_epoch_physical_time = Epoch::now().physical_time();
 
         for sst in &reclaimed_level.table_infos {
-            let unmatched_sst = sst
-                .key_range
-                .as_ref()
-                .unwrap()
-                .sstable_overlap(&state.last_select_end_bound);
+            let unmatched_sst = sst.key_range.sstable_overlap(&state.last_select_end_bound);
 
             if unmatched_sst
                 || level_handler.is_pending_compact(&sst.sst_id)
@@ -173,9 +170,9 @@ impl TtlReclaimCompactionPicker {
 
         let select_last_sst = select_input_ssts.last().unwrap();
         state.last_select_end_bound.full_key_extend(&KeyRange {
-            left: vec![],
-            right: select_last_sst.key_range.as_ref().unwrap().right.clone(),
-            right_exclusive: select_last_sst.key_range.as_ref().unwrap().right_exclusive,
+            left: Bytes::default(),
+            right: select_last_sst.key_range.right.clone(),
+            right_exclusive: select_last_sst.key_range.right_exclusive,
         });
 
         Some(CompactionInput {
@@ -205,8 +202,10 @@ mod test {
     use std::sync::Arc;
 
     use itertools::Itertools;
+    use risingwave_hummock_sdk::level::Level;
+    use risingwave_hummock_sdk::version::HummockVersionStateTableInfo;
     use risingwave_pb::hummock::compact_task;
-    pub use risingwave_pb::hummock::{Level, LevelType};
+    pub use risingwave_pb::hummock::LevelType;
 
     use super::*;
     use crate::hummock::compaction::compaction_config::CompactionConfigBuilder;
@@ -217,6 +216,7 @@ mod test {
     use crate::hummock::compaction::selector::{CompactionSelector, TtlCompactionSelector};
     use crate::hummock::compaction::{CompactionDeveloperConfig, LocalSelectorStatistic};
     use crate::hummock::model::CompactionGroup;
+    use crate::hummock::test_utils::compaction_selector_context;
 
     #[test]
     fn test_ttl_reclaim_compaction_selector() {
@@ -245,7 +245,7 @@ mod test {
             ),
             Level {
                 level_idx: 4,
-                level_type: LevelType::Nonoverlapping as i32,
+                level_type: LevelType::Nonoverlapping,
                 table_infos: vec![
                     generate_table_with_ids_and_epochs(2, 1, 0, 100, 1, vec![2], expired_epoch, 0),
                     generate_table_with_ids_and_epochs(
@@ -348,14 +348,14 @@ mod test {
 
         {
             let sst_10 = levels[3].table_infos.get_mut(8).unwrap();
-            assert_eq!(10, sst_10.get_sst_id());
-            sst_10.key_range.as_mut().unwrap().right_exclusive = true;
+            assert_eq!(10, sst_10.sst_id);
+            sst_10.key_range.right_exclusive = true;
         }
 
         assert_eq!(levels.len(), 4);
         let levels = Levels {
             levels,
-            l0: Some(l0),
+            l0,
             ..Default::default()
         };
         let mut levels_handler = (0..5).map(LevelHandler::new).collect_vec();
@@ -376,13 +376,17 @@ mod test {
             let task = selector
                 .pick_compaction(
                     1,
-                    &group_config,
-                    &levels,
-                    &BTreeSet::new(),
-                    &mut levels_handler,
-                    &mut local_stats,
-                    table_id_to_options,
-                    Arc::new(CompactionDeveloperConfig::default()),
+                    compaction_selector_context(
+                        &group_config,
+                        &levels,
+                        &BTreeSet::new(),
+                        &mut levels_handler,
+                        &mut local_stats,
+                        &table_id_to_options,
+                        Arc::new(CompactionDeveloperConfig::default()),
+                        &Default::default(),
+                        &HummockVersionStateTableInfo::empty(),
+                    ),
                 )
                 .unwrap();
             assert_compaction_task(&task, &levels_handler);
@@ -392,7 +396,7 @@ mod test {
 
             let mut start_id = 2;
             for sst in &task.input.input_levels[0].table_infos {
-                assert_eq!(start_id, sst.get_sst_id());
+                assert_eq!(start_id, sst.sst_id);
                 start_id += 1;
             }
 
@@ -427,13 +431,17 @@ mod test {
             let task = selector
                 .pick_compaction(
                     1,
-                    &group_config,
-                    &levels,
-                    &BTreeSet::new(),
-                    &mut levels_handler,
-                    &mut local_stats,
-                    table_id_to_options.clone(),
-                    Arc::new(CompactionDeveloperConfig::default()),
+                    compaction_selector_context(
+                        &group_config,
+                        &levels,
+                        &BTreeSet::new(),
+                        &mut levels_handler,
+                        &mut local_stats,
+                        &table_id_to_options,
+                        Arc::new(CompactionDeveloperConfig::default()),
+                        &Default::default(),
+                        &HummockVersionStateTableInfo::empty(),
+                    ),
                 )
                 .unwrap();
             assert_compaction_task(&task, &levels_handler);
@@ -445,7 +453,7 @@ mod test {
 
             let mut start_id = 3;
             for sst in &task.input.input_levels[0].table_infos {
-                assert_eq!(start_id, sst.get_sst_id());
+                assert_eq!(start_id, sst.sst_id);
                 start_id += 1;
             }
 
@@ -461,13 +469,17 @@ mod test {
             let task = selector
                 .pick_compaction(
                     1,
-                    &group_config,
-                    &levels,
-                    &BTreeSet::new(),
-                    &mut levels_handler,
-                    &mut local_stats,
-                    table_id_to_options.clone(),
-                    Arc::new(CompactionDeveloperConfig::default()),
+                    compaction_selector_context(
+                        &group_config,
+                        &levels,
+                        &BTreeSet::new(),
+                        &mut levels_handler,
+                        &mut local_stats,
+                        &table_id_to_options,
+                        Arc::new(CompactionDeveloperConfig::default()),
+                        &Default::default(),
+                        &HummockVersionStateTableInfo::empty(),
+                    ),
                 )
                 .unwrap();
             assert_compaction_task(&task, &levels_handler);
@@ -482,7 +494,7 @@ mod test {
                 compact_task::TaskType::Ttl
             ));
             for sst in &task.input.input_levels[0].table_infos {
-                assert_eq!(start_id, sst.get_sst_id());
+                assert_eq!(start_id, sst.sst_id);
                 start_id += 1;
             }
         }
@@ -518,13 +530,17 @@ mod test {
             let task = selector
                 .pick_compaction(
                     1,
-                    &group_config,
-                    &levels,
-                    &BTreeSet::new(),
-                    &mut levels_handler,
-                    &mut local_stats,
-                    table_id_to_options,
-                    Arc::new(CompactionDeveloperConfig::default()),
+                    compaction_selector_context(
+                        &group_config,
+                        &levels,
+                        &BTreeSet::new(),
+                        &mut levels_handler,
+                        &mut local_stats,
+                        &table_id_to_options,
+                        Arc::new(CompactionDeveloperConfig::default()),
+                        &Default::default(),
+                        &HummockVersionStateTableInfo::empty(),
+                    ),
                 )
                 .unwrap();
             assert_compaction_task(&task, &levels_handler);
@@ -534,7 +550,7 @@ mod test {
             // test table_option_filter
             assert_eq!(task.input.input_levels[0].table_infos.len(), 1);
             let select_sst = &task.input.input_levels[0].table_infos.first().unwrap();
-            assert_eq!(select_sst.get_sst_id(), 5);
+            assert_eq!(select_sst.sst_id, 5);
 
             assert_eq!(task.input.input_levels[1].level_idx, 4);
             assert_eq!(task.input.input_levels[1].table_infos.len(), 0);
@@ -560,13 +576,17 @@ mod test {
             // // pick ttl reclaim
             let task = selector.pick_compaction(
                 1,
-                &group_config,
-                &levels,
-                &BTreeSet::new(),
-                &mut levels_handler,
-                &mut local_stats,
-                HashMap::default(),
-                Arc::new(CompactionDeveloperConfig::default()),
+                compaction_selector_context(
+                    &group_config,
+                    &levels,
+                    &BTreeSet::new(),
+                    &mut levels_handler,
+                    &mut local_stats,
+                    &HashMap::default(),
+                    Arc::new(CompactionDeveloperConfig::default()),
+                    &Default::default(),
+                    &HummockVersionStateTableInfo::empty(),
+                ),
             );
 
             // empty table_options does not select any files
@@ -623,13 +643,17 @@ mod test {
                 let task = selector
                     .pick_compaction(
                         1,
-                        &group_config,
-                        &levels,
-                        &BTreeSet::new(),
-                        &mut levels_handler,
-                        &mut local_stats,
-                        table_id_to_options.clone(),
-                        Arc::new(CompactionDeveloperConfig::default()),
+                        compaction_selector_context(
+                            &group_config,
+                            &levels,
+                            &BTreeSet::new(),
+                            &mut levels_handler,
+                            &mut local_stats,
+                            &table_id_to_options,
+                            Arc::new(CompactionDeveloperConfig::default()),
+                            &Default::default(),
+                            &HummockVersionStateTableInfo::empty(),
+                        ),
                     )
                     .unwrap();
 
@@ -642,7 +666,7 @@ mod test {
                 let select_sst = &task.input.input_levels[0]
                     .table_infos
                     .iter()
-                    .map(|sst| sst.get_sst_id())
+                    .map(|sst| sst.sst_id)
                     .collect_vec();
                 assert!(select_sst.is_sorted());
                 assert_eq!(expect_task_sst_id_range[index], *select_sst);
@@ -716,13 +740,17 @@ mod test {
                 let task = selector
                     .pick_compaction(
                         1,
-                        &group_config,
-                        &levels,
-                        &BTreeSet::new(),
-                        &mut levels_handler,
-                        &mut local_stats,
-                        table_id_to_options.clone(),
-                        Arc::new(CompactionDeveloperConfig::default()),
+                        compaction_selector_context(
+                            &group_config,
+                            &levels,
+                            &BTreeSet::new(),
+                            &mut levels_handler,
+                            &mut local_stats,
+                            &table_id_to_options,
+                            Arc::new(CompactionDeveloperConfig::default()),
+                            &Default::default(),
+                            &HummockVersionStateTableInfo::empty(),
+                        ),
                     )
                     .unwrap();
 
@@ -735,7 +763,7 @@ mod test {
                 let select_sst = &task.input.input_levels[0]
                     .table_infos
                     .iter()
-                    .map(|sst| sst.get_sst_id())
+                    .map(|sst| sst.sst_id)
                     .collect_vec();
                 assert!(select_sst.is_sorted());
                 assert_eq!(expect_task_sst_id_range[index], *select_sst);
diff --git a/src/meta/src/hummock/compaction/picker/vnode_watermark_picker.rs b/src/meta/src/hummock/compaction/picker/vnode_watermark_picker.rs
new file mode 100644
index 0000000000000..50a33c7d42e4f
--- /dev/null
+++ b/src/meta/src/hummock/compaction/picker/vnode_watermark_picker.rs
@@ -0,0 +1,232 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::BTreeMap;
+
+use risingwave_common::catalog::TableId;
+use risingwave_hummock_sdk::key::{FullKey, TableKey};
+use risingwave_hummock_sdk::level::{InputLevel, Levels};
+use risingwave_hummock_sdk::sstable_info::SstableInfo;
+use risingwave_hummock_sdk::table_watermark::ReadTableWatermark;
+
+use crate::hummock::compaction::picker::CompactionInput;
+use crate::hummock::level_handler::LevelHandler;
+pub struct VnodeWatermarkCompactionPicker {}
+
+impl VnodeWatermarkCompactionPicker {
+    pub fn new() -> Self {
+        Self {}
+    }
+
+    /// The current implementation only picks trivial reclaim task for the bottommost level.
+    /// Must modify [`crate::hummock::compaction::CompactStatus::is_trivial_reclaim`], if nontrivial reclaim is supported in the future.
+    pub fn pick_compaction(
+        &mut self,
+        levels: &Levels,
+        level_handlers: &[LevelHandler],
+        table_watermarks: &BTreeMap<TableId, ReadTableWatermark>,
+    ) -> Option<CompactionInput> {
+        let level = levels.levels.last()?;
+        let mut select_input_ssts = vec![];
+        for sst_info in &level.table_infos {
+            if !level_handlers[level.level_idx as usize].is_pending_compact(&sst_info.sst_id)
+                && should_delete_sst_by_watermark(sst_info, table_watermarks)
+            {
+                select_input_ssts.push(sst_info.clone());
+            }
+        }
+        if select_input_ssts.is_empty() {
+            return None;
+        }
+        Some(CompactionInput {
+            select_input_size: select_input_ssts.iter().map(|sst| sst.file_size).sum(),
+            total_file_count: select_input_ssts.len() as u64,
+            input_levels: vec![
+                InputLevel {
+                    level_idx: level.level_idx,
+                    level_type: level.level_type,
+                    table_infos: select_input_ssts,
+                },
+                InputLevel {
+                    level_idx: level.level_idx,
+                    level_type: level.level_type,
+                    table_infos: vec![],
+                },
+            ],
+            target_level: level.level_idx as usize,
+            target_sub_level_id: level.sub_level_id,
+            ..Default::default()
+        })
+    }
+}
+
+fn should_delete_sst_by_watermark(
+    sst_info: &SstableInfo,
+    table_watermarks: &BTreeMap<TableId, ReadTableWatermark>,
+) -> bool {
+    // Both table id and vnode must be identical for both the left and right keys in a SST.
+    // As more data is written to the bottommost level, they will eventually become identical.
+    let left_key = FullKey::decode(&sst_info.key_range.left);
+    let right_key = FullKey::decode(&sst_info.key_range.right);
+    if left_key.user_key.table_id != right_key.user_key.table_id {
+        return false;
+    }
+    if left_key.user_key.table_key.vnode_part() != right_key.user_key.table_key.vnode_part() {
+        return false;
+    }
+    let Some(watermarks) = table_watermarks.get(&left_key.user_key.table_id) else {
+        return false;
+    };
+    should_delete_key_by_watermark(&left_key.user_key.table_key, watermarks)
+        && should_delete_key_by_watermark(&right_key.user_key.table_key, watermarks)
+}
+
+fn should_delete_key_by_watermark(
+    table_key: &TableKey<&[u8]>,
+    watermark: &ReadTableWatermark,
+) -> bool {
+    let (vnode, key) = table_key.split_vnode();
+    let Some(w) = watermark.vnode_watermarks.get(&vnode) else {
+        return false;
+    };
+    watermark.direction.filter_by_watermark(key, w)
+}
+
+#[cfg(test)]
+mod tests {
+    use bytes::{BufMut, Bytes, BytesMut};
+    use risingwave_common::hash::VirtualNode;
+    use risingwave_hummock_sdk::key::{FullKey, TableKey};
+    use risingwave_hummock_sdk::key_range::KeyRange;
+    use risingwave_hummock_sdk::sstable_info::SstableInfo;
+    use risingwave_hummock_sdk::table_watermark::{ReadTableWatermark, WatermarkDirection};
+
+    use crate::hummock::compaction::picker::vnode_watermark_picker::should_delete_sst_by_watermark;
+
+    #[test]
+    fn test_should_delete_sst_by_watermark() {
+        let table_watermarks = maplit::btreemap! {
+            1.into() => ReadTableWatermark {
+                direction: WatermarkDirection::Ascending,
+                vnode_watermarks: maplit::btreemap! {
+                    VirtualNode::from_index(16) => "some_watermark_key_8".into(),
+                    VirtualNode::from_index(17) => "some_watermark_key_8".into(),
+                },
+            },
+        };
+        let table_key = |vnode_part: usize, key_part: &str| {
+            let mut builder = BytesMut::new();
+            builder.put_slice(&VirtualNode::from_index(vnode_part).to_be_bytes());
+            builder.put_slice(&Bytes::copy_from_slice(key_part.as_bytes()));
+            TableKey(builder.freeze())
+        };
+
+        let sst_info = SstableInfo {
+            object_id: 1,
+            sst_id: 1,
+            key_range: KeyRange {
+                left: FullKey::new(2.into(), table_key(16, "some_watermark_key_1"), 0)
+                    .encode()
+                    .into(),
+                right: FullKey::new(2.into(), table_key(16, "some_watermark_key_2"), 0)
+                    .encode()
+                    .into(),
+                right_exclusive: true,
+            },
+            table_ids: vec![2],
+            ..Default::default()
+        };
+        assert!(
+            !should_delete_sst_by_watermark(&sst_info, &table_watermarks),
+            "should fail because no matching watermark found"
+        );
+
+        let sst_info = SstableInfo {
+            object_id: 1,
+            sst_id: 1,
+            key_range: KeyRange {
+                left: FullKey::new(1.into(), table_key(13, "some_watermark_key_1"), 0)
+                    .encode()
+                    .into(),
+                right: FullKey::new(1.into(), table_key(14, "some_watermark_key_2"), 0)
+                    .encode()
+                    .into(),
+                right_exclusive: true,
+            },
+            table_ids: vec![1],
+            ..Default::default()
+        };
+        assert!(
+            !should_delete_sst_by_watermark(&sst_info, &table_watermarks),
+            "should fail because no matching vnode found"
+        );
+
+        let sst_info = SstableInfo {
+            object_id: 1,
+            sst_id: 1,
+            key_range: KeyRange {
+                left: FullKey::new(1.into(), table_key(16, "some_watermark_key_1"), 0)
+                    .encode()
+                    .into(),
+                right: FullKey::new(1.into(), table_key(17, "some_watermark_key_2"), 0)
+                    .encode()
+                    .into(),
+                right_exclusive: true,
+            },
+            table_ids: vec![1],
+            ..Default::default()
+        };
+        assert!(
+            !should_delete_sst_by_watermark(&sst_info, &table_watermarks),
+            "should fail because different vnodes found"
+        );
+
+        let sst_info = SstableInfo {
+            object_id: 1,
+            sst_id: 1,
+            key_range: KeyRange {
+                left: FullKey::new(1.into(), table_key(16, "some_watermark_key_1"), 0)
+                    .encode()
+                    .into(),
+                right: FullKey::new(1.into(), table_key(16, "some_watermark_key_9"), 0)
+                    .encode()
+                    .into(),
+                right_exclusive: true,
+            },
+            table_ids: vec![1],
+            ..Default::default()
+        };
+        assert!(
+            !should_delete_sst_by_watermark(&sst_info, &table_watermarks),
+            "should fail because right key is greater than watermark"
+        );
+
+        let sst_info = SstableInfo {
+            object_id: 1,
+            sst_id: 1,
+            key_range: KeyRange {
+                left: FullKey::new(1.into(), table_key(16, "some_watermark_key_1"), 0)
+                    .encode()
+                    .into(),
+                right: FullKey::new(1.into(), table_key(16, "some_watermark_key_2"), 0)
+                    .encode()
+                    .into(),
+                right_exclusive: true,
+            },
+            table_ids: vec![1],
+            ..Default::default()
+        };
+        assert!(should_delete_sst_by_watermark(&sst_info, &table_watermarks));
+    }
+}
diff --git a/src/meta/src/hummock/compaction/selector/emergency_selector.rs b/src/meta/src/hummock/compaction/selector/emergency_selector.rs
index 685ab1487d51c..b9df77c430573 100644
--- a/src/meta/src/hummock/compaction/selector/emergency_selector.rs
+++ b/src/meta/src/hummock/compaction/selector/emergency_selector.rs
@@ -12,21 +12,13 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::HashMap;
-use std::sync::Arc;
-
-use risingwave_common::catalog::TableOption;
 use risingwave_hummock_sdk::HummockCompactionTaskId;
 use risingwave_pb::hummock::compact_task;
-use risingwave_pb::hummock::hummock_version::Levels;
 
-use super::{CompactionSelector, DynamicLevelSelectorCore, LocalSelectorStatistic};
+use super::{CompactionSelector, DynamicLevelSelectorCore};
 use crate::hummock::compaction::picker::{EmergencyCompactionPicker, LocalPickerStatistic};
-use crate::hummock::compaction::{
-    create_compaction_task, CompactionDeveloperConfig, CompactionTask,
-};
-use crate::hummock::level_handler::LevelHandler;
-use crate::hummock::model::CompactionGroup;
+use crate::hummock::compaction::selector::CompactionSelectorContext;
+use crate::hummock::compaction::{create_compaction_task, CompactionTask};
 
 #[derive(Default)]
 pub struct EmergencySelector {}
@@ -35,14 +27,16 @@ impl CompactionSelector for EmergencySelector {
     fn pick_compaction(
         &mut self,
         task_id: HummockCompactionTaskId,
-        group: &CompactionGroup,
-        levels: &Levels,
-        _member_table_ids: &std::collections::BTreeSet<risingwave_common::catalog::TableId>,
-        level_handlers: &mut [LevelHandler],
-        selector_stats: &mut LocalSelectorStatistic,
-        _table_id_to_options: HashMap<u32, TableOption>,
-        developer_config: Arc<CompactionDeveloperConfig>,
+        context: CompactionSelectorContext<'_>,
     ) -> Option<CompactionTask> {
+        let CompactionSelectorContext {
+            group,
+            levels,
+            level_handlers,
+            selector_stats,
+            developer_config,
+            ..
+        } = context;
         let dynamic_level_core = DynamicLevelSelectorCore::new(
             group.compaction_config.clone(),
             developer_config.clone(),
diff --git a/src/meta/src/hummock/compaction/selector/level_selector.rs b/src/meta/src/hummock/compaction/selector/level_selector.rs
index 38d1e35e22502..7076096fcfe16 100644
--- a/src/meta/src/hummock/compaction/selector/level_selector.rs
+++ b/src/meta/src/hummock/compaction/selector/level_selector.rs
@@ -16,14 +16,12 @@
 // This source code is licensed under both the GPLv2 (found in the
 // COPYING file in the root directory) and Apache 2.0 License
 // (found in the LICENSE.Apache file in the root directory).
-use std::collections::HashMap;
 use std::sync::Arc;
 
-use risingwave_common::catalog::TableOption;
-use risingwave_hummock_sdk::compaction_group::hummock_version_ext::HummockLevelsExt;
+use risingwave_hummock_sdk::level::Levels;
 use risingwave_hummock_sdk::HummockCompactionTaskId;
-use risingwave_pb::hummock::hummock_version::Levels;
-use risingwave_pb::hummock::{compact_task, CompactionConfig, LevelType};
+use risingwave_pb::hummock::compact_task::PbTaskType;
+use risingwave_pb::hummock::{CompactionConfig, LevelType};
 
 use super::{
     create_compaction_task, CompactionSelector, LevelCompactionPicker, TierCompactionPicker,
@@ -33,11 +31,11 @@ use crate::hummock::compaction::picker::{
     CompactionPicker, CompactionTaskValidator, IntraCompactionPicker, LocalPickerStatistic,
     MinOverlappingPicker,
 };
+use crate::hummock::compaction::selector::CompactionSelectorContext;
 use crate::hummock::compaction::{
-    create_overlap_strategy, CompactionDeveloperConfig, CompactionTask, LocalSelectorStatistic,
+    create_overlap_strategy, CompactionDeveloperConfig, CompactionTask,
 };
 use crate::hummock::level_handler::LevelHandler;
-use crate::hummock::model::CompactionGroup;
 
 pub const SCORE_BASE: u64 = 100;
 
@@ -209,8 +207,6 @@ impl DynamicLevelSelectorCore {
 
         let idle_file_count = levels
             .l0
-            .as_ref()
-            .unwrap()
             .sub_levels
             .iter()
             .map(|level| level.table_infos.len())
@@ -225,11 +221,9 @@ impl DynamicLevelSelectorCore {
             // to calculate the score
             let overlapping_file_count = levels
                 .l0
-                .as_ref()
-                .unwrap()
                 .sub_levels
                 .iter()
-                .filter(|level| level.level_type() == LevelType::Overlapping)
+                .filter(|level| level.level_type == LevelType::Overlapping)
                 .map(|level| level.table_infos.len())
                 .sum::<usize>();
             if overlapping_file_count > 0 {
@@ -252,13 +246,11 @@ impl DynamicLevelSelectorCore {
             // calculation rule to avoid unbalanced compact task due to large size.
             let total_size = levels
                 .l0
-                .as_ref()
-                .unwrap()
                 .sub_levels
                 .iter()
                 .filter(|level| {
                     level.vnode_partition_count == self.config.split_weight_by_vnode
-                        && level.level_type() == LevelType::Nonoverlapping
+                        && level.level_type == LevelType::Nonoverlapping
                 })
                 .map(|level| level.total_file_size)
                 .sum::<u64>()
@@ -272,11 +264,9 @@ impl DynamicLevelSelectorCore {
             // level count limit
             let non_overlapping_level_count = levels
                 .l0
-                .as_ref()
-                .unwrap()
                 .sub_levels
                 .iter()
-                .filter(|level| level.level_type() == LevelType::Nonoverlapping)
+                .filter(|level| level.level_type == LevelType::Nonoverlapping)
                 .count() as u64;
             let non_overlapping_level_score = non_overlapping_level_count * SCORE_BASE
                 / std::cmp::max(
@@ -359,8 +349,6 @@ impl DynamicLevelSelectorCore {
         let mut compact_to_next_level_bytes = 0;
         let l0_size = levels
             .l0
-            .as_ref()
-            .unwrap()
             .sub_levels
             .iter()
             .map(|sub_level| sub_level.total_file_size)
@@ -377,7 +365,7 @@ impl DynamicLevelSelectorCore {
         let mut level_bytes;
         let mut next_level_bytes = 0;
         for level in &levels.levels[ctx.base_level - 1..levels.levels.len()] {
-            let level_index = level.get_level_idx() as usize;
+            let level_index = level.level_idx as usize;
 
             if next_level_bytes > 0 {
                 level_bytes = next_level_bytes;
@@ -420,14 +408,16 @@ impl CompactionSelector for DynamicLevelSelector {
     fn pick_compaction(
         &mut self,
         task_id: HummockCompactionTaskId,
-        compaction_group: &CompactionGroup,
-        levels: &Levels,
-        _member_table_ids: &std::collections::BTreeSet<risingwave_common::catalog::TableId>,
-        level_handlers: &mut [LevelHandler],
-        selector_stats: &mut LocalSelectorStatistic,
-        _table_id_to_options: HashMap<u32, TableOption>,
-        developer_config: Arc<CompactionDeveloperConfig>,
+        context: CompactionSelectorContext<'_>,
     ) -> Option<CompactionTask> {
+        let CompactionSelectorContext {
+            group: compaction_group,
+            levels,
+            level_handlers,
+            selector_stats,
+            developer_config,
+            ..
+        } = context;
         let dynamic_level_core = DynamicLevelSelectorCore::new(
             compaction_group.compaction_config.clone(),
             developer_config,
@@ -472,8 +462,8 @@ impl CompactionSelector for DynamicLevelSelector {
         "DynamicLevelSelector"
     }
 
-    fn task_type(&self) -> compact_task::TaskType {
-        compact_task::TaskType::Dynamic
+    fn task_type(&self) -> PbTaskType {
+        PbTaskType::Dynamic
     }
 }
 
@@ -484,8 +474,9 @@ pub mod tests {
 
     use itertools::Itertools;
     use risingwave_common::constants::hummock::CompactionFilterFlag;
+    use risingwave_hummock_sdk::level::Levels;
+    use risingwave_hummock_sdk::version::HummockVersionStateTableInfo;
     use risingwave_pb::hummock::compaction_config::CompactionMode;
-    use risingwave_pb::hummock::hummock_version::Levels;
 
     use crate::hummock::compaction::compaction_config::CompactionConfigBuilder;
     use crate::hummock::compaction::selector::tests::{
@@ -498,6 +489,7 @@ pub mod tests {
     use crate::hummock::compaction::CompactionDeveloperConfig;
     use crate::hummock::level_handler::LevelHandler;
     use crate::hummock::model::CompactionGroup;
+    use crate::hummock::test_utils::compaction_selector_context;
 
     #[test]
     fn test_dynamic_level() {
@@ -521,7 +513,7 @@ pub mod tests {
         ];
         let mut levels = Levels {
             levels,
-            l0: Some(generate_l0_nonoverlapping_sublevels(vec![])),
+            l0: generate_l0_nonoverlapping_sublevels(vec![]),
             ..Default::default()
         };
         let ctx = selector.calculate_level_base_size(&levels);
@@ -557,8 +549,8 @@ pub mod tests {
         assert_eq!(ctx.level_max_bytes[3], 600);
         assert_eq!(ctx.level_max_bytes[4], 3000);
 
-        levels.l0.as_mut().unwrap().sub_levels.clear();
-        levels.l0.as_mut().unwrap().total_file_size = 0;
+        levels.l0.sub_levels.clear();
+        levels.l0.total_file_size = 0;
         levels.levels[0].table_infos = generate_tables(26..32, 0..1000, 1, 100);
         levels.levels[0].total_file_size = levels.levels[0]
             .table_infos
@@ -595,12 +587,7 @@ pub mod tests {
         ];
         let mut levels = Levels {
             levels,
-            l0: Some(generate_l0_nonoverlapping_sublevels(generate_tables(
-                15..25,
-                0..600,
-                3,
-                10,
-            ))),
+            l0: generate_l0_nonoverlapping_sublevels(generate_tables(15..25, 0..600, 3, 10)),
             ..Default::default()
         };
 
@@ -610,13 +597,17 @@ pub mod tests {
         let compaction = selector
             .pick_compaction(
                 1,
-                &group_config,
-                &levels,
-                &BTreeSet::new(),
-                &mut levels_handlers,
-                &mut local_stats,
-                HashMap::default(),
-                Arc::new(CompactionDeveloperConfig::default()),
+                compaction_selector_context(
+                    &group_config,
+                    &levels,
+                    &BTreeSet::new(),
+                    &mut levels_handlers,
+                    &mut local_stats,
+                    &HashMap::default(),
+                    Arc::new(CompactionDeveloperConfig::default()),
+                    &Default::default(),
+                    &HummockVersionStateTableInfo::empty(),
+                ),
             )
             .unwrap();
         assert_compaction_task(&compaction, &levels_handlers);
@@ -631,20 +622,24 @@ pub mod tests {
         let group_config = CompactionGroup::new(1, config.clone());
         let mut selector = DynamicLevelSelector::default();
 
-        levels.l0.as_mut().unwrap().sub_levels.clear();
-        levels.l0.as_mut().unwrap().total_file_size = 0;
+        levels.l0.sub_levels.clear();
+        levels.l0.total_file_size = 0;
         push_tables_level0_nonoverlapping(&mut levels, generate_tables(15..25, 0..600, 3, 20));
         let mut levels_handlers = (0..5).map(LevelHandler::new).collect_vec();
         let compaction = selector
             .pick_compaction(
                 1,
-                &group_config,
-                &levels,
-                &BTreeSet::new(),
-                &mut levels_handlers,
-                &mut local_stats,
-                HashMap::default(),
-                Arc::new(CompactionDeveloperConfig::default()),
+                compaction_selector_context(
+                    &group_config,
+                    &levels,
+                    &BTreeSet::new(),
+                    &mut levels_handlers,
+                    &mut local_stats,
+                    &HashMap::default(),
+                    Arc::new(CompactionDeveloperConfig::default()),
+                    &Default::default(),
+                    &HummockVersionStateTableInfo::empty(),
+                ),
             )
             .unwrap();
         assert_compaction_task(&compaction, &levels_handlers);
@@ -653,18 +648,22 @@ pub mod tests {
 
         levels_handlers[0].remove_task(1);
         levels_handlers[2].remove_task(1);
-        levels.l0.as_mut().unwrap().sub_levels.clear();
+        levels.l0.sub_levels.clear();
         levels.levels[1].table_infos = generate_tables(20..30, 0..1000, 3, 10);
         let compaction = selector
             .pick_compaction(
                 2,
-                &group_config,
-                &levels,
-                &BTreeSet::new(),
-                &mut levels_handlers,
-                &mut local_stats,
-                HashMap::default(),
-                Arc::new(CompactionDeveloperConfig::default()),
+                compaction_selector_context(
+                    &group_config,
+                    &levels,
+                    &BTreeSet::new(),
+                    &mut levels_handlers,
+                    &mut local_stats,
+                    &HashMap::default(),
+                    Arc::new(CompactionDeveloperConfig::default()),
+                    &Default::default(),
+                    &HummockVersionStateTableInfo::empty(),
+                ),
             )
             .unwrap();
         assert_compaction_task(&compaction, &levels_handlers);
@@ -674,7 +673,7 @@ pub mod tests {
             compaction.input.input_levels[0]
                 .table_infos
                 .iter()
-                .map(|sst| sst.get_sst_id())
+                .map(|sst| sst.sst_id)
                 .collect_vec(),
             vec![5]
         );
@@ -682,7 +681,7 @@ pub mod tests {
             compaction.input.input_levels[1]
                 .table_infos
                 .iter()
-                .map(|sst| sst.get_sst_id())
+                .map(|sst| sst.sst_id)
                 .collect_vec(),
             vec![10]
         );
@@ -695,13 +694,17 @@ pub mod tests {
         // to score.
         let compaction = selector.pick_compaction(
             2,
-            &group_config,
-            &levels,
-            &BTreeSet::new(),
-            &mut levels_handlers,
-            &mut local_stats,
-            HashMap::default(),
-            Arc::new(CompactionDeveloperConfig::default()),
+            compaction_selector_context(
+                &group_config,
+                &levels,
+                &BTreeSet::new(),
+                &mut levels_handlers,
+                &mut local_stats,
+                &HashMap::default(),
+                Arc::new(CompactionDeveloperConfig::default()),
+                &Default::default(),
+                &HummockVersionStateTableInfo::empty(),
+            ),
         );
         assert!(compaction.is_none());
     }
@@ -722,12 +725,7 @@ pub mod tests {
         ];
         let levels = Levels {
             levels,
-            l0: Some(generate_l0_nonoverlapping_sublevels(generate_tables(
-                15..25,
-                0..600,
-                3,
-                100,
-            ))),
+            l0: generate_l0_nonoverlapping_sublevels(generate_tables(15..25, 0..600, 3, 100)),
             ..Default::default()
         };
 
@@ -737,7 +735,7 @@ pub mod tests {
         );
         let ctx = dynamic_level_core.calculate_level_base_size(&levels);
         assert_eq!(1, ctx.base_level);
-        assert_eq!(1000, levels.l0.as_ref().unwrap().total_file_size); // l0
+        assert_eq!(1000, levels.l0.total_file_size); // l0
         assert_eq!(0, levels.levels.first().unwrap().total_file_size); // l1
         assert_eq!(25000, levels.levels.get(1).unwrap().total_file_size); // l2
         assert_eq!(15000, levels.levels.get(2).unwrap().total_file_size); // l3
diff --git a/src/meta/src/hummock/compaction/selector/manual_selector.rs b/src/meta/src/hummock/compaction/selector/manual_selector.rs
index 62c94c8f888df..82f9cbfc3e256 100644
--- a/src/meta/src/hummock/compaction/selector/manual_selector.rs
+++ b/src/meta/src/hummock/compaction/selector/manual_selector.rs
@@ -17,24 +17,20 @@
 // COPYING file in the root directory) and Apache 2.0 License
 // (found in the LICENSE.Apache file in the root directory).
 
-use std::collections::{HashMap, HashSet};
-use std::sync::Arc;
+use std::collections::HashSet;
 
-use risingwave_common::catalog::TableOption;
+use bytes::Bytes;
 use risingwave_hummock_sdk::compaction_group::StateTableId;
+use risingwave_hummock_sdk::key_range::KeyRange;
 use risingwave_hummock_sdk::{HummockCompactionTaskId, HummockSstableId};
-use risingwave_pb::hummock::hummock_version::Levels;
-use risingwave_pb::hummock::{compact_task, KeyRange};
+use risingwave_pb::hummock::compact_task;
 
-use super::{CompactionSelector, DynamicLevelSelectorCore, LocalSelectorStatistic};
+use super::{CompactionSelector, DynamicLevelSelectorCore};
 use crate::hummock::compaction::picker::{
     CompactionPicker, LocalPickerStatistic, ManualCompactionPicker,
 };
-use crate::hummock::compaction::{
-    create_compaction_task, create_overlap_strategy, CompactionDeveloperConfig, CompactionTask,
-};
-use crate::hummock::level_handler::LevelHandler;
-use crate::hummock::model::CompactionGroup;
+use crate::hummock::compaction::selector::CompactionSelectorContext;
+use crate::hummock::compaction::{create_compaction_task, create_overlap_strategy, CompactionTask};
 
 #[derive(Clone, Debug, PartialEq)]
 pub struct ManualCompactionOption {
@@ -53,8 +49,8 @@ impl Default for ManualCompactionOption {
         Self {
             sst_ids: vec![],
             key_range: KeyRange {
-                left: vec![],
-                right: vec![],
+                left: Bytes::default(),
+                right: Bytes::default(),
                 right_exclusive: false,
             },
             internal_table_id: HashSet::default(),
@@ -77,14 +73,15 @@ impl CompactionSelector for ManualCompactionSelector {
     fn pick_compaction(
         &mut self,
         task_id: HummockCompactionTaskId,
-        group: &CompactionGroup,
-        levels: &Levels,
-        _member_table_ids: &std::collections::BTreeSet<risingwave_common::catalog::TableId>,
-        level_handlers: &mut [LevelHandler],
-        _selector_stats: &mut LocalSelectorStatistic,
-        _table_id_to_options: HashMap<u32, TableOption>,
-        developer_config: Arc<CompactionDeveloperConfig>,
+        context: CompactionSelectorContext<'_>,
     ) -> Option<CompactionTask> {
+        let CompactionSelectorContext {
+            group,
+            levels,
+            level_handlers,
+            developer_config,
+            ..
+        } = context;
         let dynamic_level_core =
             DynamicLevelSelectorCore::new(group.compaction_config.clone(), developer_config);
         let overlap_strategy = create_overlap_strategy(group.compaction_config.compaction_mode());
diff --git a/src/meta/src/hummock/compaction/selector/mod.rs b/src/meta/src/hummock/compaction/selector/mod.rs
index aca2457da62dc..9c813efa6c864 100644
--- a/src/meta/src/hummock/compaction/selector/mod.rs
+++ b/src/meta/src/hummock/compaction/selector/mod.rs
@@ -23,6 +23,7 @@ mod manual_selector;
 mod space_reclaim_selector;
 mod tombstone_compaction_selector;
 mod ttl_selector;
+mod vnode_watermark_selector;
 
 use std::collections::{BTreeSet, HashMap};
 use std::sync::Arc;
@@ -31,12 +32,15 @@ pub use emergency_selector::EmergencySelector;
 pub use level_selector::{DynamicLevelSelector, DynamicLevelSelectorCore};
 pub use manual_selector::{ManualCompactionOption, ManualCompactionSelector};
 use risingwave_common::catalog::{TableId, TableOption};
+use risingwave_hummock_sdk::level::Levels;
+use risingwave_hummock_sdk::table_watermark::TableWatermarks;
+use risingwave_hummock_sdk::version::HummockVersionStateTableInfo;
 use risingwave_hummock_sdk::HummockCompactionTaskId;
 use risingwave_pb::hummock::compact_task;
-use risingwave_pb::hummock::hummock_version::Levels;
 pub use space_reclaim_selector::SpaceReclaimCompactionSelector;
 pub use tombstone_compaction_selector::TombstoneCompactionSelector;
 pub use ttl_selector::TtlCompactionSelector;
+pub use vnode_watermark_selector::VnodeWatermarkCompactionSelector;
 
 use super::picker::LocalPickerStatistic;
 use super::{
@@ -47,17 +51,23 @@ use crate::hummock::level_handler::LevelHandler;
 use crate::hummock::model::CompactionGroup;
 use crate::rpc::metrics::MetaMetrics;
 
+pub struct CompactionSelectorContext<'a> {
+    pub group: &'a CompactionGroup,
+    pub levels: &'a Levels,
+    pub member_table_ids: &'a BTreeSet<TableId>,
+    pub level_handlers: &'a mut [LevelHandler],
+    pub selector_stats: &'a mut LocalSelectorStatistic,
+    pub table_id_to_options: &'a HashMap<u32, TableOption>,
+    pub developer_config: Arc<CompactionDeveloperConfig>,
+    pub table_watermarks: &'a HashMap<TableId, Arc<TableWatermarks>>,
+    pub state_table_info: &'a HummockVersionStateTableInfo,
+}
+
 pub trait CompactionSelector: Sync + Send {
     fn pick_compaction(
         &mut self,
         task_id: HummockCompactionTaskId,
-        group: &CompactionGroup,
-        levels: &Levels,
-        member_table_ids: &BTreeSet<TableId>,
-        level_handlers: &mut [LevelHandler],
-        selector_stats: &mut LocalSelectorStatistic,
-        table_id_to_options: HashMap<u32, TableOption>,
-        developer_config: Arc<CompactionDeveloperConfig>,
+        context: CompactionSelectorContext<'_>,
     ) -> Option<CompactionTask>;
 
     fn report_statistic_metrics(&self, _metrics: &MetaMetrics) {}
@@ -117,19 +127,22 @@ pub mod tests {
     use std::ops::Range;
 
     use itertools::Itertools;
-    use risingwave_pb::hummock::{KeyRange, Level, LevelType, OverlappingLevel, SstableInfo};
+    use risingwave_hummock_sdk::key_range::KeyRange;
+    use risingwave_hummock_sdk::level::{Level, OverlappingLevel};
+    use risingwave_hummock_sdk::sstable_info::SstableInfo;
+    use risingwave_pb::hummock::LevelType;
 
     use super::*;
     use crate::hummock::test_utils::iterator_test_key_of_epoch;
 
     pub fn push_table_level0_overlapping(levels: &mut Levels, sst: SstableInfo) {
-        levels.l0.as_mut().unwrap().total_file_size += sst.file_size;
-        levels.l0.as_mut().unwrap().sub_levels.push(Level {
+        levels.l0.total_file_size += sst.file_size;
+        levels.l0.sub_levels.push(Level {
             level_idx: 0,
-            level_type: LevelType::Overlapping as i32,
+            level_type: LevelType::Overlapping,
             total_file_size: sst.file_size,
             uncompressed_file_size: sst.uncompressed_file_size,
-            sub_level_id: sst.get_sst_id(),
+            sub_level_id: sst.sst_id,
             table_infos: vec![sst],
             ..Default::default()
         });
@@ -137,14 +150,7 @@ pub mod tests {
 
     pub fn push_table_level0_nonoverlapping(levels: &mut Levels, sst: SstableInfo) {
         push_table_level0_overlapping(levels, sst);
-        levels
-            .l0
-            .as_mut()
-            .unwrap()
-            .sub_levels
-            .last_mut()
-            .unwrap()
-            .level_type = LevelType::Nonoverlapping as i32;
+        levels.l0.sub_levels.last_mut().unwrap().level_type = LevelType::Nonoverlapping;
     }
 
     pub fn push_tables_level0_nonoverlapping(levels: &mut Levels, table_infos: Vec<SstableInfo>) {
@@ -153,11 +159,11 @@ pub mod tests {
             .iter()
             .map(|table| table.uncompressed_file_size)
             .sum();
-        let sub_level_id = table_infos[0].get_sst_id();
-        levels.l0.as_mut().unwrap().total_file_size += total_file_size;
-        levels.l0.as_mut().unwrap().sub_levels.push(Level {
+        let sub_level_id = table_infos[0].sst_id;
+        levels.l0.total_file_size += total_file_size;
+        levels.l0.sub_levels.push(Level {
             level_idx: 0,
-            level_type: LevelType::Nonoverlapping as i32,
+            level_type: LevelType::Nonoverlapping,
             total_file_size,
             sub_level_id,
             table_infos,
@@ -176,11 +182,11 @@ pub mod tests {
         SstableInfo {
             object_id: id,
             sst_id: id,
-            key_range: Some(KeyRange {
-                left: iterator_test_key_of_epoch(table_prefix, left, epoch),
-                right: iterator_test_key_of_epoch(table_prefix, right, epoch),
+            key_range: KeyRange {
+                left: iterator_test_key_of_epoch(table_prefix, left, epoch).into(),
+                right: iterator_test_key_of_epoch(table_prefix, right, epoch).into(),
                 right_exclusive: false,
-            }),
+            },
             file_size: (right - left + 1) as u64,
             table_ids: vec![table_prefix as u32],
             uncompressed_file_size: (right - left + 1) as u64,
@@ -203,11 +209,11 @@ pub mod tests {
         SstableInfo {
             object_id: id,
             sst_id: id,
-            key_range: Some(KeyRange {
-                left: iterator_test_key_of_epoch(table_prefix, left, epoch),
-                right: iterator_test_key_of_epoch(table_prefix, right, epoch),
+            key_range: KeyRange {
+                left: iterator_test_key_of_epoch(table_prefix, left, epoch).into(),
+                right: iterator_test_key_of_epoch(table_prefix, right, epoch).into(),
                 right_exclusive: false,
-            }),
+            },
             file_size: (right - left + 1) as u64,
             table_ids,
             uncompressed_file_size: (right - left + 1) as u64,
@@ -243,7 +249,7 @@ pub mod tests {
             .sum();
         Level {
             level_idx,
-            level_type: LevelType::Nonoverlapping as i32,
+            level_type: LevelType::Nonoverlapping,
             table_infos,
             total_file_size,
             sub_level_id: 0,
@@ -266,7 +272,7 @@ pub mod tests {
                 .enumerate()
                 .map(|(idx, table)| Level {
                     level_idx: 0,
-                    level_type: LevelType::Nonoverlapping as i32,
+                    level_type: LevelType::Nonoverlapping,
                     total_file_size: table.file_size,
                     uncompressed_file_size: table.uncompressed_file_size,
                     sub_level_id: idx as u64,
@@ -288,7 +294,7 @@ pub mod tests {
                 .enumerate()
                 .map(|(idx, table)| Level {
                     level_idx: 0,
-                    level_type: LevelType::Nonoverlapping as i32,
+                    level_type: LevelType::Nonoverlapping,
                     total_file_size: table.iter().map(|table| table.file_size).sum::<u64>(),
                     uncompressed_file_size: table
                         .iter()
@@ -323,7 +329,7 @@ pub mod tests {
                 .enumerate()
                 .map(|(idx, table)| Level {
                     level_idx: 0,
-                    level_type: LevelType::Overlapping as i32,
+                    level_type: LevelType::Overlapping,
                     total_file_size: table.iter().map(|table| table.file_size).sum::<u64>(),
                     sub_level_id: idx as u64,
                     table_infos: table.clone(),
diff --git a/src/meta/src/hummock/compaction/selector/space_reclaim_selector.rs b/src/meta/src/hummock/compaction/selector/space_reclaim_selector.rs
index b284a6a538b3e..7f0eb0137c96b 100644
--- a/src/meta/src/hummock/compaction/selector/space_reclaim_selector.rs
+++ b/src/meta/src/hummock/compaction/selector/space_reclaim_selector.rs
@@ -17,21 +17,15 @@
 // COPYING file in the root directory) and Apache 2.0 License
 // (found in the LICENSE.Apache file in the root directory).
 
-use std::collections::{BTreeSet, HashMap};
-use std::sync::Arc;
+use std::collections::HashMap;
 
-use risingwave_common::catalog::{TableId, TableOption};
 use risingwave_hummock_sdk::HummockCompactionTaskId;
 use risingwave_pb::hummock::compact_task;
-use risingwave_pb::hummock::hummock_version::Levels;
 
 use super::{CompactionSelector, DynamicLevelSelectorCore};
 use crate::hummock::compaction::picker::{SpaceReclaimCompactionPicker, SpaceReclaimPickerState};
-use crate::hummock::compaction::{
-    create_compaction_task, CompactionDeveloperConfig, CompactionTask, LocalSelectorStatistic,
-};
-use crate::hummock::level_handler::LevelHandler;
-use crate::hummock::model::CompactionGroup;
+use crate::hummock::compaction::selector::CompactionSelectorContext;
+use crate::hummock::compaction::{create_compaction_task, CompactionTask};
 
 #[derive(Default)]
 pub struct SpaceReclaimCompactionSelector {
@@ -42,14 +36,16 @@ impl CompactionSelector for SpaceReclaimCompactionSelector {
     fn pick_compaction(
         &mut self,
         task_id: HummockCompactionTaskId,
-        group: &CompactionGroup,
-        levels: &Levels,
-        member_table_ids: &BTreeSet<TableId>,
-        level_handlers: &mut [LevelHandler],
-        _selector_stats: &mut LocalSelectorStatistic,
-        _table_id_to_options: HashMap<u32, TableOption>,
-        developer_config: Arc<CompactionDeveloperConfig>,
+        context: CompactionSelectorContext<'_>,
     ) -> Option<CompactionTask> {
+        let CompactionSelectorContext {
+            group,
+            levels,
+            member_table_ids,
+            level_handlers,
+            developer_config,
+            ..
+        } = context;
         let dynamic_level_core =
             DynamicLevelSelectorCore::new(group.compaction_config.clone(), developer_config);
         let mut picker = SpaceReclaimCompactionPicker::new(
diff --git a/src/meta/src/hummock/compaction/selector/tombstone_compaction_selector.rs b/src/meta/src/hummock/compaction/selector/tombstone_compaction_selector.rs
index b05802513733d..860bcd47c0a8d 100644
--- a/src/meta/src/hummock/compaction/selector/tombstone_compaction_selector.rs
+++ b/src/meta/src/hummock/compaction/selector/tombstone_compaction_selector.rs
@@ -13,23 +13,16 @@
 // limitations under the License.
 
 use std::collections::HashMap;
-use std::sync::Arc;
 
-use risingwave_common::catalog::TableOption;
 use risingwave_hummock_sdk::HummockCompactionTaskId;
 use risingwave_pb::hummock::compact_task;
-use risingwave_pb::hummock::hummock_version::Levels;
 
 use super::{CompactionSelector, DynamicLevelSelectorCore};
 use crate::hummock::compaction::picker::{
     TombstoneReclaimCompactionPicker, TombstoneReclaimPickerState,
 };
-use crate::hummock::compaction::{
-    create_compaction_task, create_overlap_strategy, CompactionDeveloperConfig, CompactionTask,
-    LocalSelectorStatistic,
-};
-use crate::hummock::level_handler::LevelHandler;
-use crate::hummock::model::CompactionGroup;
+use crate::hummock::compaction::selector::CompactionSelectorContext;
+use crate::hummock::compaction::{create_compaction_task, create_overlap_strategy, CompactionTask};
 
 #[derive(Default)]
 pub struct TombstoneCompactionSelector {
@@ -40,14 +33,15 @@ impl CompactionSelector for TombstoneCompactionSelector {
     fn pick_compaction(
         &mut self,
         task_id: HummockCompactionTaskId,
-        group: &CompactionGroup,
-        levels: &Levels,
-        _member_table_ids: &std::collections::BTreeSet<risingwave_common::catalog::TableId>,
-        level_handlers: &mut [LevelHandler],
-        _selector_stats: &mut LocalSelectorStatistic,
-        _table_id_to_options: HashMap<u32, TableOption>,
-        developer_config: Arc<CompactionDeveloperConfig>,
+        context: CompactionSelectorContext<'_>,
     ) -> Option<CompactionTask> {
+        let CompactionSelectorContext {
+            group,
+            levels,
+            level_handlers,
+            developer_config,
+            ..
+        } = context;
         if group.compaction_config.tombstone_reclaim_ratio == 0 {
             // it might cause full-compaction when tombstone_reclaim_ratio == 0
             return None;
diff --git a/src/meta/src/hummock/compaction/selector/ttl_selector.rs b/src/meta/src/hummock/compaction/selector/ttl_selector.rs
index 0e9497b06b17d..c20ce3304b777 100644
--- a/src/meta/src/hummock/compaction/selector/ttl_selector.rs
+++ b/src/meta/src/hummock/compaction/selector/ttl_selector.rs
@@ -18,20 +18,14 @@
 // (found in the LICENSE.Apache file in the root directory).
 
 use std::collections::HashMap;
-use std::sync::Arc;
 
-use risingwave_common::catalog::TableOption;
 use risingwave_hummock_sdk::HummockCompactionTaskId;
 use risingwave_pb::hummock::compact_task;
-use risingwave_pb::hummock::hummock_version::Levels;
 
 use super::{CompactionSelector, DynamicLevelSelectorCore};
 use crate::hummock::compaction::picker::{TtlPickerState, TtlReclaimCompactionPicker};
-use crate::hummock::compaction::{
-    create_compaction_task, CompactionDeveloperConfig, CompactionTask, LocalSelectorStatistic,
-};
-use crate::hummock::level_handler::LevelHandler;
-use crate::hummock::model::CompactionGroup;
+use crate::hummock::compaction::selector::CompactionSelectorContext;
+use crate::hummock::compaction::{create_compaction_task, CompactionTask};
 
 #[derive(Default)]
 pub struct TtlCompactionSelector {
@@ -42,14 +36,16 @@ impl CompactionSelector for TtlCompactionSelector {
     fn pick_compaction(
         &mut self,
         task_id: HummockCompactionTaskId,
-        group: &CompactionGroup,
-        levels: &Levels,
-        _member_table_ids: &std::collections::BTreeSet<risingwave_common::catalog::TableId>,
-        level_handlers: &mut [LevelHandler],
-        _selector_stats: &mut LocalSelectorStatistic,
-        table_id_to_options: HashMap<u32, TableOption>,
-        developer_config: Arc<CompactionDeveloperConfig>,
+        context: CompactionSelectorContext<'_>,
     ) -> Option<CompactionTask> {
+        let CompactionSelectorContext {
+            group,
+            levels,
+            level_handlers,
+            table_id_to_options,
+            developer_config,
+            ..
+        } = context;
         let dynamic_level_core =
             DynamicLevelSelectorCore::new(group.compaction_config.clone(), developer_config);
         let ctx = dynamic_level_core.calculate_level_base_size(levels);
diff --git a/src/meta/src/hummock/compaction/selector/vnode_watermark_selector.rs b/src/meta/src/hummock/compaction/selector/vnode_watermark_selector.rs
new file mode 100644
index 0000000000000..e09ed7e661581
--- /dev/null
+++ b/src/meta/src/hummock/compaction/selector/vnode_watermark_selector.rs
@@ -0,0 +1,87 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::{BTreeMap, BTreeSet, HashMap};
+use std::sync::Arc;
+
+use risingwave_common::catalog::TableId;
+use risingwave_hummock_sdk::compaction_group::hummock_version_ext::{
+    safe_epoch_read_table_watermarks_impl, safe_epoch_table_watermarks_impl,
+};
+use risingwave_hummock_sdk::table_watermark::{ReadTableWatermark, TableWatermarks};
+use risingwave_hummock_sdk::version::HummockVersionStateTableInfo;
+use risingwave_hummock_sdk::HummockCompactionTaskId;
+use risingwave_pb::hummock::compact_task::TaskType;
+
+use crate::hummock::compaction::picker::VnodeWatermarkCompactionPicker;
+use crate::hummock::compaction::selector::{CompactionSelectorContext, DynamicLevelSelectorCore};
+use crate::hummock::compaction::{create_compaction_task, CompactionSelector, CompactionTask};
+#[derive(Default)]
+pub struct VnodeWatermarkCompactionSelector {}
+
+impl CompactionSelector for VnodeWatermarkCompactionSelector {
+    fn pick_compaction(
+        &mut self,
+        task_id: HummockCompactionTaskId,
+        context: CompactionSelectorContext<'_>,
+    ) -> Option<CompactionTask> {
+        let CompactionSelectorContext {
+            group,
+            levels,
+            level_handlers,
+            developer_config,
+            table_watermarks,
+            member_table_ids,
+            state_table_info,
+            ..
+        } = context;
+        let dynamic_level_core =
+            DynamicLevelSelectorCore::new(group.compaction_config.clone(), developer_config);
+        let ctx = dynamic_level_core.calculate_level_base_size(levels);
+        let mut picker = VnodeWatermarkCompactionPicker::new();
+        let table_watermarks =
+            safe_epoch_read_table_watermarks(table_watermarks, state_table_info, member_table_ids);
+        let compaction_input = picker.pick_compaction(levels, level_handlers, &table_watermarks)?;
+        compaction_input.add_pending_task(task_id, level_handlers);
+        Some(create_compaction_task(
+            dynamic_level_core.get_config(),
+            compaction_input,
+            ctx.base_level,
+            self.task_type(),
+        ))
+    }
+
+    fn name(&self) -> &'static str {
+        "VnodeWatermarkCompaction"
+    }
+
+    fn task_type(&self) -> TaskType {
+        TaskType::VnodeWatermark
+    }
+}
+
+fn safe_epoch_read_table_watermarks(
+    table_watermarks: &HashMap<TableId, Arc<TableWatermarks>>,
+    state_table_info: &HummockVersionStateTableInfo,
+    member_table_ids: &BTreeSet<TableId>,
+) -> BTreeMap<TableId, ReadTableWatermark> {
+    safe_epoch_read_table_watermarks_impl(&safe_epoch_table_watermarks_impl(
+        table_watermarks,
+        state_table_info,
+        &member_table_ids
+            .iter()
+            .map(TableId::table_id)
+            .collect::<Vec<_>>(),
+    ))
+}
diff --git a/src/meta/src/hummock/compactor_manager.rs b/src/meta/src/hummock/compactor_manager.rs
index ab0c868f703c4..252f92c404015 100644
--- a/src/meta/src/hummock/compactor_manager.rs
+++ b/src/meta/src/hummock/compactor_manager.rs
@@ -19,11 +19,11 @@ use std::time::{Duration, Instant, SystemTime};
 use fail::fail_point;
 use parking_lot::RwLock;
 use risingwave_hummock_sdk::compact::statistics_compact_task;
+use risingwave_hummock_sdk::compact_task::CompactTask;
 use risingwave_hummock_sdk::{HummockCompactionTaskId, HummockContextId};
 use risingwave_pb::hummock::subscribe_compaction_event_response::Event as ResponseEvent;
 use risingwave_pb::hummock::{
-    CancelCompactTask, CompactTask, CompactTaskAssignment, CompactTaskProgress,
-    SubscribeCompactionEventResponse,
+    CancelCompactTask, CompactTaskAssignment, CompactTaskProgress, SubscribeCompactionEventResponse,
 };
 use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender};
 
@@ -156,7 +156,7 @@ impl CompactorManagerInner {
         };
         // Initialize heartbeat for existing tasks.
         task_assignment.into_iter().for_each(|assignment| {
-            manager.initiate_task_heartbeat(assignment.compact_task.unwrap());
+            manager.initiate_task_heartbeat(CompactTask::from(assignment.compact_task.unwrap()));
         });
         Ok(manager)
     }
@@ -302,7 +302,7 @@ impl CompactorManagerInner {
                         task.target_level,
                         task.base_level,
                         task.target_sub_level_id,
-                        task.task_type,
+                        task.task_type.as_str_name(),
                         compact_task_statistics
                 );
             }
diff --git a/src/meta/src/hummock/error.rs b/src/meta/src/hummock/error.rs
index 434de47623310..af82cc4690b04 100644
--- a/src/meta/src/hummock/error.rs
+++ b/src/meta/src/hummock/error.rs
@@ -27,7 +27,7 @@ pub type Result<T> = std::result::Result<T, Error>;
 pub enum Error {
     #[error("invalid hummock context {0}")]
     InvalidContext(HummockContextId),
-    #[error("failed to access meta store: {0}")]
+    #[error("failed to access meta store")]
     MetaStore(
         #[source]
         #[backtrace]
@@ -45,6 +45,12 @@ pub enum Error {
     CompactionGroup(String),
     #[error("SST {0} is invalid")]
     InvalidSst(HummockSstableObjectId),
+    #[error("time travel")]
+    TimeTravel(
+        #[source]
+        #[backtrace]
+        anyhow::Error,
+    ),
     #[error(transparent)]
     Internal(
         #[from]
diff --git a/src/meta/src/hummock/level_handler.rs b/src/meta/src/hummock/level_handler.rs
index 25bc8af8e66d2..2f729b7123d57 100644
--- a/src/meta/src/hummock/level_handler.rs
+++ b/src/meta/src/hummock/level_handler.rs
@@ -15,9 +15,10 @@
 use std::collections::HashMap;
 
 use itertools::Itertools;
+use risingwave_hummock_sdk::level::Level;
+use risingwave_hummock_sdk::sstable_info::SstableInfo;
 use risingwave_hummock_sdk::{HummockCompactionTaskId, HummockSstableId};
 use risingwave_pb::hummock::level_handler::RunningCompactTask;
-use risingwave_pb::hummock::{Level, SstableInfo};
 
 #[derive(Clone, Debug, PartialEq)]
 pub struct LevelHandler {
@@ -90,9 +91,9 @@ impl LevelHandler {
         let mut table_ids = vec![];
         let mut total_file_size = 0;
         for sst in ssts {
-            self.compacting_files.insert(sst.get_sst_id(), task_id);
+            self.compacting_files.insert(sst.sst_id, task_id);
             total_file_size += sst.file_size;
-            table_ids.push(sst.get_sst_id());
+            table_ids.push(sst.sst_id);
         }
 
         self.pending_tasks.push(RunningCompactTask {
diff --git a/src/meta/src/hummock/manager/checkpoint.rs b/src/meta/src/hummock/manager/checkpoint.rs
index 70bbef6bd3db2..bf63d3eed9690 100644
--- a/src/meta/src/hummock/manager/checkpoint.rs
+++ b/src/meta/src/hummock/manager/checkpoint.rs
@@ -22,10 +22,10 @@ use risingwave_hummock_sdk::compaction_group::hummock_version_ext::{
 };
 use risingwave_hummock_sdk::version::HummockVersion;
 use risingwave_hummock_sdk::HummockVersionId;
-use risingwave_pb::hummock::hummock_version_checkpoint::{
-    StaleObjects as PbStaleObjects, StaleObjects,
+use risingwave_pb::hummock::hummock_version_checkpoint::{PbStaleObjects, StaleObjects};
+use risingwave_pb::hummock::{
+    PbHummockVersion, PbHummockVersionArchive, PbHummockVersionCheckpoint,
 };
-use risingwave_pb::hummock::{PbHummockVersionArchive, PbHummockVersionCheckpoint};
 use thiserror_ext::AsReport;
 use tracing::warn;
 
@@ -60,7 +60,7 @@ impl HummockVersionCheckpoint {
 
     pub fn to_protobuf(&self) -> PbHummockVersionCheckpoint {
         PbHummockVersionCheckpoint {
-            version: Some(self.version.to_protobuf()),
+            version: Some(PbHummockVersion::from(&self.version)),
             stale_objects: self.stale_objects.clone(),
         }
     }
@@ -197,14 +197,20 @@ impl HummockManager {
         );
         if self.env.opts.enable_hummock_data_archive {
             archive = Some(PbHummockVersionArchive {
-                version: Some(old_checkpoint.version.to_protobuf()),
+                version: Some(PbHummockVersion::from(&old_checkpoint.version)),
                 version_deltas: versioning
                     .hummock_version_deltas
                     .range((Excluded(old_checkpoint_id), Included(new_checkpoint_id)))
-                    .map(|(_, version_delta)| version_delta.to_protobuf())
+                    .map(|(_, version_delta)| version_delta.into())
                     .collect(),
             });
         }
+        // Whenever data archive or time travel is enabled, we can directly discard reference to stale objects that will no longer be used.
+        if self.env.opts.enable_hummock_data_archive || self.time_travel_enabled().await {
+            let context_info = self.context_info.read().await;
+            let min_pinned_version_id = context_info.min_pinned_version_id();
+            stale_objects.retain(|version_id, _| *version_id >= min_pinned_version_id);
+        }
         let new_checkpoint = HummockVersionCheckpoint {
             version: current_version.clone(),
             stale_objects,
@@ -227,8 +233,8 @@ impl HummockManager {
         let context_info = self.context_info.read().await;
         assert!(new_checkpoint.version.id > versioning.checkpoint.version.id);
         versioning.checkpoint = new_checkpoint;
-        // Not delete stale objects when archive is enabled
-        if !self.env.opts.enable_hummock_data_archive {
+        // Not delete stale objects when archive or time travel is enabled
+        if !self.env.opts.enable_hummock_data_archive && !self.time_travel_enabled().await {
             versioning.mark_objects_for_deletion(&context_info, &self.delete_object_tracker);
         }
 
diff --git a/src/meta/src/hummock/manager/commit_epoch.rs b/src/meta/src/hummock/manager/commit_epoch.rs
index 9a494bc509b4f..2054bd4dbb5d2 100644
--- a/src/meta/src/hummock/manager/commit_epoch.rs
+++ b/src/meta/src/hummock/manager/commit_epoch.rs
@@ -14,22 +14,25 @@
 
 use std::collections::{BTreeMap, HashMap, HashSet};
 
-use itertools::Itertools;
 use risingwave_common::catalog::TableId;
+use risingwave_hummock_sdk::change_log::ChangeLogDelta;
+use risingwave_hummock_sdk::compaction_group::hummock_version_ext::split_sst;
 use risingwave_hummock_sdk::compaction_group::StaticCompactionGroupId;
+use risingwave_hummock_sdk::sstable_info::SstableInfo;
 use risingwave_hummock_sdk::table_stats::{
-    add_prost_table_stats_map, purge_prost_table_stats, PbTableStatsMap,
+    add_prost_table_stats_map, purge_prost_table_stats, to_prost_table_stats_map, PbTableStatsMap,
 };
 use risingwave_hummock_sdk::table_watermark::TableWatermarks;
+use risingwave_hummock_sdk::version::HummockVersionStateTableInfo;
 use risingwave_hummock_sdk::{
-    CompactionGroupId, ExtendedSstableInfo, HummockContextId, HummockEpoch, HummockSstableObjectId,
+    CompactionGroupId, HummockContextId, HummockEpoch, HummockSstableObjectId, LocalSstableInfo,
 };
 use risingwave_pb::hummock::compact_task::{self};
-use risingwave_pb::hummock::group_delta::DeltaType;
-use risingwave_pb::hummock::hummock_version_delta::ChangeLogDelta;
-use risingwave_pb::hummock::{GroupDelta, HummockSnapshot, IntraLevelDelta, StateTableInfoDelta};
+use risingwave_pb::hummock::HummockSnapshot;
+use sea_orm::TransactionTrait;
 
 use crate::hummock::error::{Error, Result};
+use crate::hummock::manager::compaction_group_manager::CompactionGroupManager;
 use crate::hummock::manager::transaction::{
     HummockVersionStatsTransaction, HummockVersionTransaction,
 };
@@ -37,8 +40,12 @@ use crate::hummock::manager::versioning::Versioning;
 use crate::hummock::metrics_utils::{
     get_or_create_local_table_stat, trigger_local_table_stat, trigger_sst_stat,
 };
-use crate::hummock::sequence::next_sstable_object_id;
-use crate::hummock::{commit_multi_var, start_measure_real_process_timer, HummockManager};
+use crate::hummock::model::CompactionGroup;
+use crate::hummock::sequence::{next_compaction_group_id, next_sstable_object_id};
+use crate::hummock::{
+    commit_multi_var, commit_multi_var_with_provided_txn, start_measure_real_process_timer,
+    HummockManager,
+};
 
 #[derive(Debug, Clone)]
 pub struct NewTableFragmentInfo {
@@ -47,25 +54,34 @@ pub struct NewTableFragmentInfo {
     pub internal_table_ids: Vec<TableId>,
 }
 
+pub struct BatchCommitForNewCg {
+    pub epoch_to_ssts: BTreeMap<HummockEpoch, Vec<LocalSstableInfo>>,
+    pub table_ids: Vec<TableId>,
+}
+
 pub struct CommitEpochInfo {
-    pub sstables: Vec<ExtendedSstableInfo>,
+    pub sstables: Vec<LocalSstableInfo>,
     pub new_table_watermarks: HashMap<TableId, TableWatermarks>,
     pub sst_to_context: HashMap<HummockSstableObjectId, HummockContextId>,
     pub new_table_fragment_info: Option<NewTableFragmentInfo>,
     pub change_log_delta: HashMap<TableId, ChangeLogDelta>,
     pub table_committed_epoch: BTreeMap<HummockEpoch, HashSet<TableId>>,
     pub max_committed_epoch: HummockEpoch,
+
+    // commit multi Epoch and SSTs for new compaction group
+    pub batch_commit_for_new_cg: Vec<BatchCommitForNewCg>,
 }
 
 impl CommitEpochInfo {
     pub fn new(
-        sstables: Vec<ExtendedSstableInfo>,
+        sstables: Vec<LocalSstableInfo>,
         new_table_watermarks: HashMap<TableId, TableWatermarks>,
         sst_to_context: HashMap<HummockSstableObjectId, HummockContextId>,
         new_table_fragment_info: Option<NewTableFragmentInfo>,
         change_log_delta: HashMap<TableId, ChangeLogDelta>,
         table_committed_epoch: BTreeMap<HummockEpoch, HashSet<TableId>>,
         max_committed_epoch: HummockEpoch,
+        batch_commit_for_new_cg: Vec<BatchCommitForNewCg>,
     ) -> Self {
         Self {
             sstables,
@@ -75,6 +91,7 @@ impl CommitEpochInfo {
             change_log_delta,
             table_committed_epoch,
             max_committed_epoch,
+            batch_commit_for_new_cg,
         }
     }
 }
@@ -84,7 +101,7 @@ impl HummockManager {
     pub async fn commit_epoch_for_test(
         &self,
         epoch: HummockEpoch,
-        sstables: Vec<impl Into<ExtendedSstableInfo>>,
+        sstables: Vec<impl Into<LocalSstableInfo>>,
         sst_to_context: HashMap<HummockSstableObjectId, HummockContextId>,
     ) -> Result<()> {
         let tables = self
@@ -105,6 +122,7 @@ impl HummockManager {
             HashMap::new(),
             BTreeMap::from_iter([(epoch, tables)]),
             epoch,
+            vec![],
         );
         self.commit_epoch(info).await?;
         Ok(())
@@ -123,6 +141,7 @@ impl HummockManager {
             change_log_delta,
             table_committed_epoch,
             max_committed_epoch: epoch,
+            batch_commit_for_new_cg,
         } = commit_info;
         let mut versioning_guard = self.versioning.write().await;
         let _timer = start_measure_real_process_timer!(self, "commit_epoch");
@@ -143,7 +162,17 @@ impl HummockManager {
         // Consume and aggregate table stats.
         let mut table_stats_change = PbTableStatsMap::default();
         for s in &mut sstables {
-            add_prost_table_stats_map(&mut table_stats_change, &std::mem::take(&mut s.table_stats));
+            add_prost_table_stats_map(
+                &mut table_stats_change,
+                &to_prost_table_stats_map(std::mem::take(&mut s.table_stats)),
+            );
+        }
+
+        let previous_time_travel_toggle_check = versioning.time_travel_toggle_check;
+        versioning.time_travel_toggle_check = self.time_travel_enabled().await;
+        if !previous_time_travel_toggle_check && versioning.time_travel_toggle_check {
+            // Take a snapshot for the first commit epoch after enabling time travel.
+            versioning.mark_next_time_travel_version_snapshot();
         }
 
         let mut version = HummockVersionTransaction::new(
@@ -152,216 +181,91 @@ impl HummockManager {
             self.env.notification_manager(),
             &self.metrics,
         );
-        let mut new_version_delta = version.new_delta();
-
-        new_version_delta.max_committed_epoch = epoch;
-        new_version_delta.new_table_watermarks = new_table_watermarks;
-        new_version_delta.change_log_delta = change_log_delta;
 
-        let mut table_compaction_group_mapping = new_version_delta
-            .latest_version()
-            .state_table_info
-            .build_table_compaction_group_id();
+        let state_table_info = &version.latest_version().state_table_info;
 
-        let mut new_table_ids = None;
+        let mut table_compaction_group_mapping = state_table_info.build_table_compaction_group_id();
 
+        let mut new_table_ids = HashMap::new();
         // Add new table
         if let Some(new_fragment_table_info) = new_table_fragment_info {
-            let new_table_ids = new_table_ids.insert(HashMap::new());
             if !new_fragment_table_info.internal_table_ids.is_empty() {
-                for table_id in &new_fragment_table_info.internal_table_ids {
-                    if let Some(info) = new_version_delta
-                        .latest_version()
-                        .state_table_info
-                        .info()
-                        .get(table_id)
-                    {
-                        return Err(Error::CompactionGroup(format!(
-                            "table {} already exist {:?}",
-                            table_id.table_id, info,
-                        )));
-                    }
-                }
-
-                for table_id in &new_fragment_table_info.internal_table_ids {
-                    table_compaction_group_mapping
-                        .insert(*table_id, StaticCompactionGroupId::StateDefault as u64);
-                    new_table_ids.insert(*table_id, StaticCompactionGroupId::StateDefault as u64);
-                }
+                on_handle_add_new_table(
+                    state_table_info,
+                    &new_fragment_table_info.internal_table_ids,
+                    StaticCompactionGroupId::StateDefault as u64,
+                    &mut table_compaction_group_mapping,
+                    &mut new_table_ids,
+                )?;
             }
 
-            if let Some(table_id) = new_fragment_table_info.mv_table_id {
-                if let Some(info) = new_version_delta
-                    .latest_version()
-                    .state_table_info
-                    .info()
-                    .get(&table_id)
-                {
-                    return Err(Error::CompactionGroup(format!(
-                        "table {} already exist {:?}",
-                        table_id.table_id, info,
-                    )));
-                }
-                let _ = table_compaction_group_mapping
-                    .insert(table_id, StaticCompactionGroupId::MaterializedView as u64);
-                new_table_ids.insert(table_id, StaticCompactionGroupId::MaterializedView as u64);
+            if let Some(mv_table_id) = new_fragment_table_info.mv_table_id {
+                on_handle_add_new_table(
+                    state_table_info,
+                    &[mv_table_id],
+                    StaticCompactionGroupId::MaterializedView as u64,
+                    &mut table_compaction_group_mapping,
+                    &mut new_table_ids,
+                )?;
             }
         }
 
-        let mut incorrect_ssts = vec![];
-        let mut new_sst_id_number = 0;
-        for ExtendedSstableInfo {
-            compaction_group_id,
-            sst_info: sst,
-            ..
-        } in &mut sstables
-        {
-            let is_sst_belong_to_group_declared = match new_version_delta
-                .latest_version()
-                .levels
-                .get(compaction_group_id)
-            {
-                Some(_compaction_group) => sst.table_ids.iter().all(|t| {
-                    table_compaction_group_mapping
-                        .get(&TableId::new(*t))
-                        .map(|table_cg_id| table_cg_id == compaction_group_id)
-                        .unwrap_or(false)
-                }),
-                None => false,
-            };
-            if !is_sst_belong_to_group_declared {
-                let mut group_table_ids: BTreeMap<_, Vec<u32>> = BTreeMap::new();
-                for table_id in sst.get_table_ids() {
-                    match table_compaction_group_mapping.get(&TableId::new(*table_id)) {
-                        Some(compaction_group_id) => {
-                            group_table_ids
-                                .entry(*compaction_group_id)
-                                .or_default()
-                                .push(*table_id);
-                        }
-                        None => {
-                            tracing::warn!(
-                                "table {} in SST {} doesn't belong to any compaction group",
-                                table_id,
-                                sst.get_object_id(),
-                            );
-                        }
-                    }
-                }
-                let is_trivial_adjust = group_table_ids.len() == 1
-                    && group_table_ids.first_key_value().unwrap().1.len()
-                        == sst.get_table_ids().len();
-                if is_trivial_adjust {
-                    *compaction_group_id = *group_table_ids.first_key_value().unwrap().0;
-                    // is_sst_belong_to_group_declared = true;
-                } else {
-                    new_sst_id_number += group_table_ids.len();
-                    incorrect_ssts.push((std::mem::take(sst), group_table_ids));
-                    *compaction_group_id =
-                        StaticCompactionGroupId::NewCompactionGroup as CompactionGroupId;
-                }
-            }
-        }
-        let mut new_sst_id = next_sstable_object_id(&self.env, new_sst_id_number).await?;
-        let original_sstables = std::mem::take(&mut sstables);
-        sstables.reserve_exact(original_sstables.len() - incorrect_ssts.len() + new_sst_id_number);
-        let mut incorrect_ssts = incorrect_ssts.into_iter();
-        for original_sstable in original_sstables {
-            if original_sstable.compaction_group_id
-                == StaticCompactionGroupId::NewCompactionGroup as CompactionGroupId
-            {
-                let (sst, group_table_ids) = incorrect_ssts.next().unwrap();
-                let mut branch_groups = HashMap::new();
-                for (group_id, _match_ids) in group_table_ids {
-                    let mut branch_sst = sst.clone();
-                    branch_sst.sst_id = new_sst_id;
-                    sstables.push(ExtendedSstableInfo::with_compaction_group(
-                        group_id, branch_sst,
-                    ));
-                    branch_groups.insert(group_id, new_sst_id);
-                    new_sst_id += 1;
+        let (batch_commit_for_new_cg, compaction_group_manager_txn) =
+            if !batch_commit_for_new_cg.is_empty() {
+                let compaction_group_manager_guard = self.compaction_group_manager.write().await;
+                let compaction_group_config =
+                    compaction_group_manager_guard.default_compaction_config();
+                let mut compaction_group_manager =
+                    CompactionGroupManager::start_owned_compaction_groups_txn(
+                        compaction_group_manager_guard,
+                    );
+                let mut batch_commit_info = HashMap::new();
+                for BatchCommitForNewCg {
+                    epoch_to_ssts,
+                    table_ids,
+                } in batch_commit_for_new_cg
+                {
+                    let new_compaction_group_id = next_compaction_group_id(&self.env).await?;
+                    compaction_group_manager.insert(
+                        new_compaction_group_id,
+                        CompactionGroup {
+                            group_id: new_compaction_group_id,
+                            compaction_config: compaction_group_config.clone(),
+                        },
+                    );
+
+                    on_handle_add_new_table(
+                        state_table_info,
+                        &table_ids,
+                        new_compaction_group_id,
+                        &mut table_compaction_group_mapping,
+                        &mut new_table_ids,
+                    )?;
+
+                    batch_commit_info.insert(new_compaction_group_id, epoch_to_ssts);
                 }
+                (
+                    Some((batch_commit_info, (*compaction_group_config).clone())),
+                    Some(compaction_group_manager),
+                )
             } else {
-                sstables.push(original_sstable);
-            }
-        }
-
-        let mut modified_compaction_groups = vec![];
-        // Append SSTs to a new version.
-        for (compaction_group_id, sstables) in &sstables
-            .into_iter()
-            // the sort is stable sort, and will not change the order within compaction group.
-            // Do a sort so that sst in the same compaction group can be consecutive
-            .sorted_by_key(
-                |ExtendedSstableInfo {
-                     compaction_group_id,
-                     ..
-                 }| *compaction_group_id,
-            )
-            .group_by(
-                |ExtendedSstableInfo {
-                     compaction_group_id,
-                     ..
-                 }| *compaction_group_id,
-            )
-        {
-            modified_compaction_groups.push(compaction_group_id);
-            let group_sstables = sstables
-                .into_iter()
-                .map(|ExtendedSstableInfo { sst_info, .. }| sst_info)
-                .collect_vec();
-
-            let group_deltas = &mut new_version_delta
-                .group_deltas
-                .entry(compaction_group_id)
-                .or_default()
-                .group_deltas;
-            let l0_sub_level_id = epoch;
-            let group_delta = GroupDelta {
-                delta_type: Some(DeltaType::IntraLevel(IntraLevelDelta {
-                    level_idx: 0,
-                    inserted_table_infos: group_sstables.clone(),
-                    l0_sub_level_id,
-                    ..Default::default()
-                })),
+                (None, None)
             };
-            group_deltas.push(group_delta);
-        }
 
-        // update state table info
-        new_version_delta.with_latest_version(|version, delta| {
-            if let Some(new_table_ids) = new_table_ids {
-                for (table_id, cg_id) in new_table_ids {
-                    delta.state_table_info_delta.insert(
-                        table_id,
-                        StateTableInfoDelta {
-                            committed_epoch: epoch,
-                            safe_epoch: epoch,
-                            compaction_group_id: cg_id,
-                        },
-                    );
-                }
-            }
-            for (table_id, info) in version.state_table_info.info() {
-                assert!(
-                    delta
-                        .state_table_info_delta
-                        .insert(
-                            *table_id,
-                            StateTableInfoDelta {
-                                committed_epoch: epoch,
-                                safe_epoch: info.safe_epoch,
-                                compaction_group_id: info.compaction_group_id,
-                            }
-                        )
-                        .is_none(),
-                    "newly added table exists previously: {:?}",
-                    table_id
-                );
-            }
-        });
+        let commit_sstables = self
+            .correct_commit_ssts(sstables, &table_compaction_group_mapping)
+            .await?;
 
-        new_version_delta.pre_apply();
+        let modified_compaction_groups: Vec<_> = commit_sstables.keys().cloned().collect();
+
+        let time_travel_delta = version.pre_commit_epoch(
+            epoch,
+            commit_sstables,
+            new_table_ids,
+            new_table_watermarks,
+            change_log_delta,
+            batch_commit_for_new_cg,
+        );
 
         // TODO: remove the sanity check when supporting partial checkpoint
         assert_eq!(1, table_committed_epoch.len());
@@ -411,7 +315,53 @@ impl HummockManager {
             );
             table_metrics.inc_write_throughput(stats_value as u64);
         }
-        commit_multi_var!(self.meta_store_ref(), version, version_stats)?;
+        if versioning.time_travel_toggle_check
+            && let Some(sql_store) = self.sql_store()
+        {
+            let mut time_travel_version = None;
+            if versioning.time_travel_snapshot_interval_counter
+                >= self.env.opts.hummock_time_travel_snapshot_interval
+            {
+                versioning.time_travel_snapshot_interval_counter = 0;
+                time_travel_version = Some(version.latest_version());
+            } else {
+                versioning.time_travel_snapshot_interval_counter = versioning
+                    .time_travel_snapshot_interval_counter
+                    .saturating_add(1);
+            }
+            let group_parents = version
+                .latest_version()
+                .levels
+                .values()
+                .map(|g| (g.group_id, g.parent_group_id))
+                .collect();
+            let mut txn = sql_store.conn.begin().await?;
+            let version_snapshot_sst_ids = self
+                .write_time_travel_metadata(
+                    &txn,
+                    time_travel_version,
+                    time_travel_delta,
+                    &group_parents,
+                    &versioning.last_time_travel_snapshot_sst_ids,
+                )
+                .await?;
+            commit_multi_var_with_provided_txn!(
+                txn,
+                version,
+                version_stats,
+                compaction_group_manager_txn
+            )?;
+            if let Some(version_snapshot_sst_ids) = version_snapshot_sst_ids {
+                versioning.last_time_travel_snapshot_sst_ids = version_snapshot_sst_ids;
+            }
+        } else {
+            commit_multi_var!(
+                self.meta_store_ref(),
+                version,
+                version_stats,
+                compaction_group_manager_txn
+            )?;
+        }
 
         let snapshot = HummockSnapshot {
             committed_epoch: epoch,
@@ -430,36 +380,15 @@ impl HummockManager {
             );
         }
 
+        drop(versioning_guard);
         tracing::trace!("new committed epoch {}", epoch);
 
-        let mut table_groups = HashMap::<u32, usize>::default();
-        for (table_id, info) in versioning.current_version.state_table_info.info() {
-            table_groups.insert(
-                table_id.table_id,
-                versioning
-                    .current_version
-                    .state_table_info
-                    .compaction_group_member_tables()
-                    .get(&info.compaction_group_id)
-                    .expect("should exist")
-                    .len(),
-            );
-        }
-        drop(versioning_guard);
         // Don't trigger compactions if we enable deterministic compaction
         if !self.env.opts.compaction_deterministic_test {
             // commit_epoch may contains SSTs from any compaction group
             for id in &modified_compaction_groups {
                 self.try_send_compaction_request(*id, compact_task::TaskType::Dynamic);
             }
-            if !table_stats_change.is_empty() {
-                table_stats_change.retain(|table_id, _| {
-                    table_groups
-                        .get(table_id)
-                        .map(|table_count| *table_count > 1)
-                        .unwrap_or(false)
-                });
-            }
             if !table_stats_change.is_empty() {
                 self.collect_table_write_throughput(table_stats_change);
             }
@@ -486,4 +415,79 @@ impl HummockManager {
             }
         }
     }
+
+    async fn correct_commit_ssts(
+        &self,
+        sstables: Vec<LocalSstableInfo>,
+        table_compaction_group_mapping: &HashMap<TableId, CompactionGroupId>,
+    ) -> Result<BTreeMap<CompactionGroupId, Vec<SstableInfo>>> {
+        let mut new_sst_id_number = 0;
+        let mut sst_to_cg_vec = Vec::with_capacity(sstables.len());
+        for commit_sst in sstables {
+            let mut group_table_ids: BTreeMap<u64, Vec<u32>> = BTreeMap::new();
+            for table_id in &commit_sst.sst_info.table_ids {
+                match table_compaction_group_mapping.get(&TableId::new(*table_id)) {
+                    Some(cg_id_from_meta) => {
+                        group_table_ids
+                            .entry(*cg_id_from_meta)
+                            .or_default()
+                            .push(*table_id);
+                    }
+                    None => {
+                        tracing::warn!(
+                            "table {} in SST {} doesn't belong to any compaction group",
+                            table_id,
+                            commit_sst.sst_info.object_id,
+                        );
+                    }
+                }
+            }
+
+            new_sst_id_number += group_table_ids.len();
+            sst_to_cg_vec.push((commit_sst, group_table_ids));
+        }
+
+        // Generate new SST IDs for each compaction group
+        // `next_sstable_object_id` will update the global SST ID and reserve the new SST IDs
+        // So we need to get the new SST ID first and then split the SSTs
+        let mut new_sst_id = next_sstable_object_id(&self.env, new_sst_id_number).await?;
+        let mut commit_sstables: BTreeMap<u64, Vec<SstableInfo>> = BTreeMap::new();
+
+        for (mut sst, group_table_ids) in sst_to_cg_vec {
+            for (group_id, _match_ids) in group_table_ids {
+                let branch_sst = split_sst(&mut sst.sst_info, &mut new_sst_id);
+                commit_sstables
+                    .entry(group_id)
+                    .or_default()
+                    .push(branch_sst);
+            }
+        }
+
+        Ok(commit_sstables)
+    }
+}
+
+fn on_handle_add_new_table(
+    state_table_info: &HummockVersionStateTableInfo,
+    table_ids: &[TableId],
+    compaction_group_id: CompactionGroupId,
+    table_compaction_group_mapping: &mut HashMap<TableId, CompactionGroupId>,
+    new_table_ids: &mut HashMap<TableId, CompactionGroupId>,
+) -> Result<()> {
+    if table_ids.is_empty() {
+        return Err(Error::CompactionGroup("empty table ids".to_string()));
+    }
+
+    for table_id in table_ids {
+        if let Some(info) = state_table_info.info().get(table_id) {
+            return Err(Error::CompactionGroup(format!(
+                "table {} already exist {:?}",
+                table_id.table_id, info,
+            )));
+        }
+        table_compaction_group_mapping.insert(*table_id, compaction_group_id);
+        new_table_ids.insert(*table_id, compaction_group_id);
+    }
+
+    Ok(())
 }
diff --git a/src/meta/src/hummock/manager/compaction.rs b/src/meta/src/hummock/manager/compaction.rs
index 906824c155f7a..b096298b7f401 100644
--- a/src/meta/src/hummock/manager/compaction.rs
+++ b/src/meta/src/hummock/manager/compaction.rs
@@ -41,29 +41,31 @@ use parking_lot::Mutex;
 use rand::seq::SliceRandom;
 use rand::thread_rng;
 use risingwave_common::util::epoch::Epoch;
+use risingwave_hummock_sdk::compact_task::{CompactTask, ReportTask};
 use risingwave_hummock_sdk::compaction_group::hummock_version_ext::HummockLevelsExt;
 use risingwave_hummock_sdk::compaction_group::StaticCompactionGroupId;
+use risingwave_hummock_sdk::key_range::KeyRange;
+use risingwave_hummock_sdk::level::{InputLevel, Level, Levels};
+use risingwave_hummock_sdk::sstable_info::SstableInfo;
 use risingwave_hummock_sdk::table_stats::{
     add_prost_table_stats_map, purge_prost_table_stats, PbTableStatsMap,
 };
-use risingwave_hummock_sdk::version::HummockVersion;
+use risingwave_hummock_sdk::version::{GroupDelta, HummockVersion, IntraLevelDelta};
 use risingwave_hummock_sdk::{
     compact_task_to_string, statistics_compact_task, CompactionGroupId, HummockCompactionTaskId,
     HummockVersionId,
 };
 use risingwave_pb::hummock::compact_task::{TaskStatus, TaskType};
-use risingwave_pb::hummock::group_delta::DeltaType;
-use risingwave_pb::hummock::hummock_version::Levels;
 use risingwave_pb::hummock::subscribe_compaction_event_request::{
-    self, Event as RequestEvent, HeartBeat, PullTask, ReportTask,
+    self, Event as RequestEvent, HeartBeat, PullTask,
 };
 use risingwave_pb::hummock::subscribe_compaction_event_response::{
     Event as ResponseEvent, PullTaskAck,
 };
 use risingwave_pb::hummock::{
-    compact_task, CompactStatus as PbCompactStatus, CompactTask, CompactTaskAssignment,
-    CompactionConfig, GroupDelta, InputLevel, IntraLevelDelta, Level, SstableInfo,
-    StateTableInfoDelta, SubscribeCompactionEventRequest, TableOption, TableSchema,
+    compact_task, CompactTaskAssignment, CompactionConfig, PbCompactStatus,
+    PbCompactTaskAssignment, StateTableInfoDelta, SubscribeCompactionEventRequest, TableOption,
+    TableSchema,
 };
 use rw_futures_util::pending_on_none;
 use thiserror_ext::AsReport;
@@ -78,7 +80,7 @@ use crate::hummock::compaction::selector::level_selector::PickerInfo;
 use crate::hummock::compaction::selector::{
     DynamicLevelSelector, DynamicLevelSelectorCore, LocalSelectorStatistic, ManualCompactionOption,
     ManualCompactionSelector, SpaceReclaimCompactionSelector, TombstoneCompactionSelector,
-    TtlCompactionSelector,
+    TtlCompactionSelector, VnodeWatermarkCompactionSelector,
 };
 use crate::hummock::compaction::{CompactStatus, CompactionDeveloperConfig, CompactionSelector};
 use crate::hummock::error::{Error, Result};
@@ -133,6 +135,10 @@ fn init_selectors() -> HashMap<compact_task::TaskType, Box<dyn CompactionSelecto
         compact_task::TaskType::Tombstone,
         Box::<TombstoneCompactionSelector>::default(),
     );
+    compaction_selectors.insert(
+        compact_task::TaskType::VnodeWatermark,
+        Box::<VnodeWatermarkCompactionSelector>::default(),
+    );
     compaction_selectors
 }
 
@@ -151,11 +157,8 @@ impl<'a> HummockVersionTransaction<'a> {
 
         for level in &compact_task.input_ssts {
             let level_idx = level.level_idx;
-            let mut removed_table_ids = level
-                .table_infos
-                .iter()
-                .map(|sst| sst.get_sst_id())
-                .collect_vec();
+            let mut removed_table_ids =
+                level.table_infos.iter().map(|sst| sst.sst_id).collect_vec();
 
             removed_table_ids_map
                 .entry(level_idx)
@@ -164,40 +167,42 @@ impl<'a> HummockVersionTransaction<'a> {
         }
 
         for (level_idx, removed_table_ids) in removed_table_ids_map {
-            let group_delta = GroupDelta {
-                delta_type: Some(DeltaType::IntraLevel(IntraLevelDelta {
-                    level_idx,
-                    removed_table_ids,
-                    ..Default::default()
-                })),
-            };
+            let group_delta = GroupDelta::IntraLevel(IntraLevelDelta::new(
+                level_idx,
+                0, // default
+                removed_table_ids,
+                vec![], // default
+                0,      // default
+            ));
+
             group_deltas.push(group_delta);
         }
 
-        let group_delta = GroupDelta {
-            delta_type: Some(DeltaType::IntraLevel(IntraLevelDelta {
-                level_idx: compact_task.target_level,
-                inserted_table_infos: compact_task.sorted_output_ssts.clone(),
-                l0_sub_level_id: compact_task.target_sub_level_id,
-                vnode_partition_count: compact_task.split_weight_by_vnode,
-                ..Default::default()
-            })),
-        };
+        let group_delta = GroupDelta::IntraLevel(IntraLevelDelta::new(
+            compact_task.target_level,
+            compact_task.target_sub_level_id,
+            vec![], // default
+            compact_task.sorted_output_ssts.clone(),
+            compact_task.split_weight_by_vnode,
+        ));
+
         group_deltas.push(group_delta);
-        version_delta.safe_epoch = std::cmp::max(
+        let new_visible_table_safe_epoch = std::cmp::max(
             version_delta.latest_version().visible_table_safe_epoch(),
             compact_task.watermark,
         );
-        if version_delta.latest_version().visible_table_safe_epoch() < version_delta.safe_epoch {
+        version_delta.set_safe_epoch(new_visible_table_safe_epoch);
+        if version_delta.latest_version().visible_table_safe_epoch() < new_visible_table_safe_epoch
+        {
             version_delta.with_latest_version(|version, version_delta| {
                 for (table_id, info) in version.state_table_info.info() {
-                    let new_safe_epoch = min(version_delta.safe_epoch, info.committed_epoch);
+                    let new_safe_epoch = min(new_visible_table_safe_epoch, info.committed_epoch);
                     if new_safe_epoch > info.safe_epoch {
-                        if new_safe_epoch != version_delta.safe_epoch {
+                        if new_safe_epoch != version_delta.visible_table_safe_epoch() {
                             warn!(
                                 new_safe_epoch,
                                 committed_epoch = info.committed_epoch,
-                                global_safe_epoch = version_delta.safe_epoch,
+                                global_safe_epoch = new_visible_table_safe_epoch,
                                 table_id = table_id.table_id,
                                 "table has different safe epoch to global"
                             );
@@ -221,7 +226,7 @@ impl<'a> HummockVersionTransaction<'a> {
 #[derive(Default)]
 pub struct Compaction {
     /// Compaction task that is already assigned to a compactor
-    pub compact_task_assignment: BTreeMap<HummockCompactionTaskId, CompactTaskAssignment>,
+    pub compact_task_assignment: BTreeMap<HummockCompactionTaskId, PbCompactTaskAssignment>,
     /// `CompactStatus` of each compaction group
     pub compaction_statuses: BTreeMap<CompactionGroupId, CompactStatus>,
 
@@ -319,7 +324,7 @@ impl HummockManager {
                             for task in compact_tasks {
                                 let task_id = task.task_id;
                                 if let Err(e) =
-                                    compactor.send_event(ResponseEvent::CompactTask(task))
+                                    compactor.send_event(ResponseEvent::CompactTask(task.into()))
                                 {
                                     tracing::warn!(
                                         error = %e.as_report(),
@@ -597,7 +602,7 @@ impl HummockManager {
                         }
 
                         RequestEvent::ReportTask(task) => {
-                           report_events.push(task);
+                           report_events.push(task.into());
                         }
 
                         _ => unreachable!(),
@@ -615,7 +620,7 @@ impl HummockManager {
                             }
 
                             RequestEvent::ReportTask(task) => {
-                                report_events.push(task);
+                                report_events.push(task.into());
                                 if report_events.len() >= MAX_REPORT_COUNT {
                                     break;
                                 }
@@ -764,8 +769,10 @@ impl HummockManager {
                 &group_config,
                 &mut stats,
                 selector,
-                table_id_to_option.clone(),
+                &table_id_to_option,
                 developer_config.clone(),
+                &version.latest_version().table_watermarks,
+                &version.latest_version().state_table_info,
             ) {
                 let target_level_id = compact_task.input.target_level as u32;
 
@@ -775,11 +782,9 @@ impl HummockManager {
                     _ => 0,
                 };
                 let vnode_partition_count = compact_task.input.vnode_partition_count;
-                use risingwave_hummock_sdk::prost_key_range::KeyRangeExt;
-
                 let mut compact_task = CompactTask {
                     input_ssts: compact_task.input.input_levels,
-                    splits: vec![risingwave_pb::hummock::KeyRange::inf()],
+                    splits: vec![KeyRange::inf()],
                     watermark,
                     sorted_output_ssts: vec![],
                     task_id,
@@ -791,7 +796,7 @@ impl HummockManager {
                         .get_compaction_group_levels(compaction_group_id)
                         .is_last_level(target_level_id),
                     base_level: compact_task.base_level as u32,
-                    task_status: TaskStatus::Pending as i32,
+                    task_status: TaskStatus::Pending,
                     compaction_group_id: group_config.group_id,
                     existing_table_ids: member_table_ids.clone(),
                     compression_algorithm,
@@ -805,7 +810,7 @@ impl HummockManager {
                     current_epoch_time: Epoch::now().0,
                     compaction_filter_mask: group_config.compaction_config.compaction_filter_mask,
                     target_sub_level_id: compact_task.input.target_sub_level_id,
-                    task_type: compact_task.compaction_task_type as i32,
+                    task_type: compact_task.compaction_task_type,
                     split_weight_by_vnode: vnode_partition_count,
                     max_sub_compaction: group_config.compaction_config.max_sub_compaction,
                     ..Default::default()
@@ -832,7 +837,7 @@ impl HummockManager {
                         compact_task.input_ssts,
                         start_time.elapsed()
                     );
-                    compact_task.set_task_status(TaskStatus::Success);
+                    compact_task.task_status = TaskStatus::Success;
                     compact_status.report_compact_task(&compact_task);
                     if !is_trivial_reclaim {
                         compact_task
@@ -886,7 +891,7 @@ impl HummockManager {
                     compact_task_assignment.insert(
                         compact_task.task_id,
                         CompactTaskAssignment {
-                            compact_task: Some(compact_task.clone()),
+                            compact_task: Some(compact_task.clone().into()),
                             context_id: META_NODE_ID, // deprecated
                         },
                     );
@@ -946,7 +951,7 @@ impl HummockManager {
                 .initiate_task_heartbeat(compact_task.clone());
 
             // this task has been finished.
-            compact_task.set_task_status(TaskStatus::Pending);
+            compact_task.task_status = TaskStatus::Pending;
             let compact_task_statistics = statistics_compact_task(compact_task);
 
             let level_type_label = build_compact_task_level_type_metrics_label(
@@ -984,7 +989,7 @@ impl HummockManager {
                     "For compaction group {}: pick up {} {} sub_level in level {} to compact to target {}. cost time: {:?} compact_task_statistics {:?}",
                     compaction_group_id,
                     level_count,
-                    compact_task.input_ssts[0].level_type().as_str_name(),
+                    compact_task.input_ssts[0].level_type.as_str_name(),
                     compact_task.input_ssts[0].level_idx,
                     compact_task.target_level,
                     start_time.elapsed(),
@@ -1029,7 +1034,7 @@ impl HummockManager {
             .into_iter()
             .map(|task_id| ReportTask {
                 task_id,
-                task_status: task_status as i32,
+                task_status,
                 sorted_output_ssts: vec![],
                 table_stats_change: HashMap::default(),
             })
@@ -1056,7 +1061,7 @@ impl HummockManager {
             .get_compact_tasks_impl(compaction_groups, max_select_count, selector)
             .await?;
         tasks.retain(|task| {
-            if task.task_status() == TaskStatus::Success {
+            if task.task_status == TaskStatus::Success {
                 debug_assert!(
                     CompactStatus::is_trivial_reclaim(task)
                         || CompactStatus::is_trivial_move_task(task)
@@ -1082,7 +1087,7 @@ impl HummockManager {
             .get_compact_tasks_impl(vec![compaction_group_id], 1, selector)
             .await?;
         for task in normal_tasks {
-            if task.task_status() != TaskStatus::Success {
+            if task.task_status != TaskStatus::Success {
                 return Ok(Some(task));
             }
             debug_assert!(
@@ -1112,7 +1117,7 @@ impl HummockManager {
             .levels
             .get(&compact_task.compaction_group_id)
         {
-            for input_level in compact_task.get_input_ssts() {
+            for input_level in &compact_task.input_ssts {
                 let input_level: &InputLevel = input_level;
                 let mut sst_ids: HashSet<_> = input_level
                     .table_infos
@@ -1125,7 +1130,7 @@ impl HummockManager {
                     }
                 }
                 if input_level.level_idx == 0 {
-                    for level in &group.get_level0().sub_levels {
+                    for level in &group.level0().sub_levels {
                         filter_ssts(level, &mut sst_ids);
                     }
                 } else {
@@ -1150,7 +1155,7 @@ impl HummockManager {
         let rets = self
             .report_compact_tasks(vec![ReportTask {
                 task_id,
-                task_status: task_status as i32,
+                task_status,
                 sorted_output_ssts,
                 table_stats_change: table_stats_change.unwrap_or_default(),
             }])
@@ -1208,7 +1213,7 @@ impl HummockManager {
         for (idx, task) in report_tasks.into_iter().enumerate() {
             rets[idx] = true;
             let mut compact_task = match compact_task_assignment.remove(task.task_id) {
-                Some(compact_task) => compact_task.compact_task.unwrap(),
+                Some(compact_task) => CompactTask::from(compact_task.compact_task.unwrap()),
                 None => {
                     tracing::warn!("{}", format!("compact task {} not found", task.task_id));
                     rets[idx] = false;
@@ -1227,7 +1232,7 @@ impl HummockManager {
                     compact_status.report_compact_task(&compact_task);
                 }
                 None => {
-                    compact_task.set_task_status(TaskStatus::InvalidGroupCanceled);
+                    compact_task.task_status = TaskStatus::InvalidGroupCanceled;
                 }
             }
 
@@ -1241,12 +1246,12 @@ impl HummockManager {
                 .iter()
                 .map(|level| level.level_idx)
                 .collect();
-            let is_success = if let TaskStatus::Success = compact_task.task_status() {
+            let is_success = if let TaskStatus::Success = compact_task.task_status {
                 // if member_table_ids changes, the data of sstable may stale.
                 let is_expired =
                     Self::is_compact_task_expired(&compact_task, version.latest_version());
                 if is_expired {
-                    compact_task.set_task_status(TaskStatus::InputOutdatedCanceled);
+                    compact_task.task_status = TaskStatus::InputOutdatedCanceled;
                     false
                 } else {
                     let group = version
@@ -1257,7 +1262,7 @@ impl HummockManager {
                     let input_exist =
                         group.check_deleted_sst_exist(&input_level_ids, input_sst_ids);
                     if !input_exist {
-                        compact_task.set_task_status(TaskStatus::InvalidGroupCanceled);
+                        compact_task.task_status = TaskStatus::InvalidGroupCanceled;
                         warn!(
                             "The task may be expired because of group split, task:\n {:?}",
                             compact_task_to_string(&compact_task)
@@ -1309,9 +1314,9 @@ impl HummockManager {
         }
         let mut success_groups = vec![];
         for compact_task in tasks {
-            let task_status = compact_task.task_status();
+            let task_status = compact_task.task_status;
             let task_status_label = task_status.as_str_name();
-            let task_type_label = compact_task.task_type().as_str_name();
+            let task_type_label = compact_task.task_type.as_str_name();
 
             self.compactor_manager
                 .remove_task_heartbeat(compact_task.task_id);
@@ -1342,8 +1347,8 @@ impl HummockManager {
             );
 
             if !deterministic_mode
-                && (matches!(compact_task.task_type(), compact_task::TaskType::Dynamic)
-                    || matches!(compact_task.task_type(), compact_task::TaskType::Emergency))
+                && (matches!(compact_task.task_type, compact_task::TaskType::Dynamic)
+                    || matches!(compact_task.task_type, compact_task::TaskType::Emergency))
             {
                 // only try send Dynamic compaction
                 self.try_send_compaction_request(
@@ -1437,7 +1442,7 @@ impl HummockManager {
         let compact_task_string = compact_task_to_string(&compact_task);
         // TODO: shall we need to cancel on meta ?
         compactor
-            .send_event(ResponseEvent::CompactTask(compact_task))
+            .send_event(ResponseEvent::CompactTask(compact_task.into()))
             .with_context(|| {
                 format!(
                     "Failed to trigger compaction task for compaction_group {}",
@@ -1664,7 +1669,7 @@ impl HummockManager {
             guard.compact_task_assignment.insert(
                 task_id,
                 CompactTaskAssignment {
-                    compact_task: Some(task),
+                    compact_task: Some(task.into()),
                     context_id: 0,
                 },
             );
@@ -1674,7 +1679,7 @@ impl HummockManager {
         // So we pass the modified compact_task directly into the `report_compact_task_impl`
         self.report_compact_tasks(vec![ReportTask {
             task_id,
-            task_status: task_status as i32,
+            task_status,
             sorted_output_ssts,
             table_stats_change: table_stats_change.unwrap_or_default(),
         }])
@@ -1688,7 +1693,7 @@ pub fn check_cg_write_limit(
     compaction_config: &CompactionConfig,
 ) -> WriteLimitType {
     let threshold = compaction_config.level0_stop_write_threshold_sub_level_number as usize;
-    let l0_sub_level_number = levels.l0.as_ref().unwrap().sub_levels.len();
+    let l0_sub_level_number = levels.l0.sub_levels.len();
     if threshold < l0_sub_level_number {
         return WriteLimitType::WriteStop(l0_sub_level_number, threshold);
     }
@@ -1766,6 +1771,8 @@ impl CompactionState {
             Some(compact_task::TaskType::Ttl)
         } else if guard.contains(&(group, compact_task::TaskType::Tombstone)) {
             Some(compact_task::TaskType::Tombstone)
+        } else if guard.contains(&(group, compact_task::TaskType::VnodeWatermark)) {
+            Some(compact_task::TaskType::VnodeWatermark)
         } else {
             None
         }
diff --git a/src/meta/src/hummock/manager/compaction_group_manager.rs b/src/meta/src/hummock/manager/compaction_group_manager.rs
index c0c00d18a01e4..d585c23e19ee1 100644
--- a/src/meta/src/hummock/manager/compaction_group_manager.rs
+++ b/src/meta/src/hummock/manager/compaction_group_manager.rs
@@ -18,21 +18,20 @@ use std::sync::Arc;
 
 use itertools::Itertools;
 use risingwave_common::catalog::TableId;
+use risingwave_hummock_sdk::compact_task::ReportTask;
 use risingwave_hummock_sdk::compaction_group::hummock_version_ext::{
     get_compaction_group_ids, TableGroupInfo,
 };
 use risingwave_hummock_sdk::compaction_group::{StateTableId, StaticCompactionGroupId};
+use risingwave_hummock_sdk::version::{GroupDelta, GroupDeltas};
 use risingwave_hummock_sdk::CompactionGroupId;
 use risingwave_meta_model_v2::compaction_config;
 use risingwave_pb::hummock::compact_task::TaskStatus;
-use risingwave_pb::hummock::group_delta::DeltaType;
-use risingwave_pb::hummock::hummock_version_delta::GroupDeltas;
 use risingwave_pb::hummock::rise_ctl_update_compaction_config_request::mutable_config::MutableConfig;
-use risingwave_pb::hummock::subscribe_compaction_event_request::ReportTask;
 use risingwave_pb::hummock::write_limits::WriteLimit;
 use risingwave_pb::hummock::{
-    compact_task, CompactionConfig, CompactionGroupInfo, CompatibilityVersion, GroupConstruct,
-    GroupDelta, GroupDestroy, StateTableInfoDelta,
+    compact_task, CompactionConfig, CompactionGroupInfo, CompatibilityVersion, PbGroupConstruct,
+    PbGroupDestroy, PbStateTableInfoDelta,
 };
 use tokio::sync::OnceCell;
 
@@ -47,7 +46,10 @@ use crate::hummock::metrics_utils::remove_compaction_group_in_sst_stat;
 use crate::hummock::model::CompactionGroup;
 use crate::hummock::sequence::{next_compaction_group_id, next_sstable_object_id};
 use crate::manager::{MetaSrvEnv, MetaStoreImpl};
-use crate::model::{BTreeMapTransaction, MetadataModel, MetadataModelError};
+use crate::model::{
+    BTreeMapTransaction, BTreeMapTransactionInner, DerefMutForward, MetadataModel,
+    MetadataModelError,
+};
 
 type CompactionGroupTransaction<'a> = BTreeMapTransaction<'a, CompactionGroupId, CompactionGroup>;
 
@@ -252,20 +254,20 @@ impl HummockManager {
                             }
                         };
 
-                    group_deltas.push(GroupDelta {
-                        delta_type: Some(DeltaType::GroupConstruct(GroupConstruct {
-                            group_config: Some(config),
-                            group_id,
-                            ..Default::default()
-                        })),
+                    let group_delta = GroupDelta::GroupConstruct(PbGroupConstruct {
+                        group_config: Some(config),
+                        group_id,
+                        ..Default::default()
                     });
+
+                    group_deltas.push(group_delta);
                 }
             }
             assert!(new_version_delta
                 .state_table_info_delta
                 .insert(
                     TableId::new(*table_id),
-                    StateTableInfoDelta {
+                    PbStateTableInfoDelta {
                         committed_epoch: epoch,
                         safe_epoch: epoch,
                         compaction_group_id: *raw_group_id,
@@ -330,7 +332,7 @@ impl HummockManager {
                         new_version_delta
                             .latest_version()
                             .get_compaction_group_levels(group_id)
-                            .get_levels()
+                            .levels
                             .len(),
                     ));
                 }
@@ -343,9 +345,9 @@ impl HummockManager {
                 .entry(*group_id)
                 .or_default()
                 .group_deltas;
-            group_deltas.push(GroupDelta {
-                delta_type: Some(DeltaType::GroupDestroy(GroupDestroy {})),
-            });
+
+            let group_delta = GroupDelta::GroupDestroy(PbGroupDestroy {});
+            group_deltas.push(group_delta);
         }
 
         for (group_id, max_level) in groups_to_remove {
@@ -363,7 +365,7 @@ impl HummockManager {
             version.latest_version(),
         )));
         commit_multi_var!(self.meta_store_ref(), version, compaction_groups_txn)?;
-
+        // No need to handle DeltaType::GroupDestroy during time travel.
         Ok(())
     }
 
@@ -532,16 +534,14 @@ impl HummockManager {
                 new_version_delta.group_deltas.insert(
                     new_compaction_group_id,
                     GroupDeltas {
-                        group_deltas: vec![GroupDelta {
-                            delta_type: Some(DeltaType::GroupConstruct(GroupConstruct {
-                                group_config: Some(config.clone()),
-                                group_id: new_compaction_group_id,
-                                parent_group_id,
-                                new_sst_start_id,
-                                table_ids: vec![],
-                                version: CompatibilityVersion::NoMemberTableIds as i32,
-                            })),
-                        }],
+                        group_deltas: vec![GroupDelta::GroupConstruct(PbGroupConstruct {
+                            group_config: Some(config.clone()),
+                            group_id: new_compaction_group_id,
+                            parent_group_id,
+                            new_sst_start_id,
+                            table_ids: vec![],
+                            version: CompatibilityVersion::NoMemberTableIds as i32,
+                        })],
                     },
                 );
                 ((new_compaction_group_id, config), new_compaction_group_id)
@@ -561,7 +561,7 @@ impl HummockManager {
                     .state_table_info_delta
                     .insert(
                         table_id,
-                        StateTableInfoDelta {
+                        PbStateTableInfoDelta {
                             committed_epoch: info.committed_epoch,
                             safe_epoch: info.safe_epoch,
                             compaction_group_id: new_compaction_group_id,
@@ -579,16 +579,19 @@ impl HummockManager {
             new_version_delta.pre_apply();
             commit_multi_var!(self.meta_store_ref(), version, compaction_groups_txn)?;
         }
-
+        // Instead of handling DeltaType::GroupConstruct for time travel, simply enforce a version snapshot.
+        versioning.mark_next_time_travel_version_snapshot();
         let mut canceled_tasks = vec![];
         for task_assignment in compaction_guard.compact_task_assignment.values() {
             if let Some(task) = task_assignment.compact_task.as_ref() {
-                let need_cancel =
-                    HummockManager::is_compact_task_expired(task, &versioning.current_version);
+                let need_cancel = HummockManager::is_compact_task_expired(
+                    &task.into(),
+                    &versioning.current_version,
+                );
                 if need_cancel {
                     canceled_tasks.push(ReportTask {
                         task_id: task.task_id,
-                        task_status: TaskStatus::ManualCanceled as i32,
+                        task_status: TaskStatus::ManualCanceled,
                         table_stats_change: HashMap::default(),
                         sorted_output_ssts: vec![],
                     });
@@ -693,6 +696,26 @@ impl CompactionGroupManager {
         CompactionGroupTransaction::new(&mut self.compaction_groups)
     }
 
+    pub fn start_owned_compaction_groups_txn<P: DerefMut<Target = Self>>(
+        inner: P,
+    ) -> BTreeMapTransactionInner<
+        CompactionGroupId,
+        CompactionGroup,
+        DerefMutForward<
+            Self,
+            BTreeMap<CompactionGroupId, CompactionGroup>,
+            P,
+            impl Fn(&Self) -> &BTreeMap<CompactionGroupId, CompactionGroup>,
+            impl Fn(&mut Self) -> &mut BTreeMap<CompactionGroupId, CompactionGroup>,
+        >,
+    > {
+        BTreeMapTransactionInner::new(DerefMutForward::new(
+            inner,
+            |mgr| &mgr.compaction_groups,
+            |mgr| &mut mgr.compaction_groups,
+        ))
+    }
+
     /// Tries to get compaction group config for `compaction_group_id`.
     pub(super) fn try_get_compaction_group_config(
         &self,
@@ -760,7 +783,7 @@ fn update_compaction_config(target: &mut CompactionConfig, items: &[MutableConfi
                     .clone_from(&c.compression_algorithm);
             }
             MutableConfig::MaxL0CompactLevelCount(c) => {
-                target.max_l0_compact_level_count = *c;
+                target.max_l0_compact_level_count = Some(*c);
             }
         }
     }
diff --git a/src/meta/src/hummock/manager/context.rs b/src/meta/src/hummock/manager/context.rs
index 982a94fd5f9db..6bfcd119b89d7 100644
--- a/src/meta/src/hummock/manager/context.rs
+++ b/src/meta/src/hummock/manager/context.rs
@@ -19,7 +19,7 @@ use itertools::Itertools;
 use risingwave_common::util::epoch::INVALID_EPOCH;
 use risingwave_hummock_sdk::version::HummockVersion;
 use risingwave_hummock_sdk::{
-    ExtendedSstableInfo, HummockContextId, HummockEpoch, HummockSstableObjectId, HummockVersionId,
+    HummockContextId, HummockEpoch, HummockSstableObjectId, HummockVersionId, LocalSstableInfo,
     INVALID_VERSION_ID,
 };
 use risingwave_pb::hummock::{
@@ -197,7 +197,7 @@ impl HummockManager {
     pub async fn commit_epoch_sanity_check(
         &self,
         epoch: HummockEpoch,
-        sstables: &[ExtendedSstableInfo],
+        sstables: &[LocalSstableInfo],
         sst_to_context: &HashMap<HummockSstableObjectId, HummockContextId>,
         current_version: &HummockVersion,
     ) -> Result<()> {
@@ -246,11 +246,11 @@ impl HummockManager {
             };
             let sst_infos = sstables
                 .iter()
-                .map(|ExtendedSstableInfo { sst_info, .. }| sst_info.clone())
+                .map(|LocalSstableInfo { sst_info, .. }| sst_info.clone())
                 .collect_vec();
             if compactor
                 .send_event(ResponseEvent::ValidationTask(ValidationTask {
-                    sst_infos,
+                    sst_infos: sst_infos.into_iter().map(|sst| sst.into()).collect_vec(),
                     sst_id_to_worker_id: sst_to_context.clone(),
                     epoch,
                 }))
diff --git a/src/meta/src/hummock/manager/gc.rs b/src/meta/src/hummock/manager/gc.rs
index 5f5150b7777cb..97a99945bcf41 100644
--- a/src/meta/src/hummock/manager/gc.rs
+++ b/src/meta/src/hummock/manager/gc.rs
@@ -177,7 +177,11 @@ impl HummockManager {
     /// 3. Meta node decides which SSTs to delete. See `HummockManager::complete_full_gc`.
     ///
     /// Returns Ok(false) if there is no worker available.
-    pub fn start_full_gc(&self, sst_retention_time: Duration) -> Result<bool> {
+    pub fn start_full_gc(
+        &self,
+        sst_retention_time: Duration,
+        prefix: Option<String>,
+    ) -> Result<bool> {
         self.metrics.full_gc_trigger_count.inc();
         // Set a minimum sst_retention_time to avoid deleting SSTs of on-going write op.
         let sst_retention_time = cmp::max(
@@ -185,8 +189,9 @@ impl HummockManager {
             Duration::from_secs(self.env.opts.min_sst_retention_time_sec),
         );
         tracing::info!(
-            "run full GC with sst_retention_time = {} secs",
-            sst_retention_time.as_secs()
+            retention_sec = sst_retention_time.as_secs(),
+            prefix = prefix.as_ref().unwrap_or(&String::from("")),
+            "run full GC"
         );
         let compactor = match self.compactor_manager.next_compactor() {
             None => {
@@ -198,6 +203,7 @@ impl HummockManager {
         compactor
             .send_event(ResponseEvent::FullScanTask(FullScanTask {
                 sst_retention_time_sec: sst_retention_time.as_secs(),
+                prefix,
             }))
             .map_err(|_| Error::CompactorUnreachable(compactor.context_id()))?;
         Ok(true)
@@ -216,23 +222,33 @@ impl HummockManager {
         let watermark =
             collect_global_gc_watermark(self.metadata_manager().clone(), spin_interval).await?;
         metrics.full_gc_last_object_id_watermark.set(watermark as _);
-        let candidate_sst_number = object_ids.len();
+        let candidate_object_number = object_ids.len();
         metrics
             .full_gc_candidate_object_count
-            .observe(candidate_sst_number as _);
+            .observe(candidate_object_number as _);
+        let pinned_object_ids = self
+            .all_object_ids_in_time_travel()
+            .await?
+            .collect::<HashSet<_>>();
         // 1. filter by watermark
         let object_ids = object_ids
             .into_iter()
             .filter(|s| *s < watermark)
             .collect_vec();
-        // 2. filter by version
-        let selected_sst_number = self.extend_objects_to_delete_from_scan(&object_ids).await;
+        let after_watermark = object_ids.len();
+        // 2. filter by time travel archive
+        let object_ids = object_ids
+            .into_iter()
+            .filter(|s| !pinned_object_ids.contains(s))
+            .collect_vec();
+        let after_time_travel = object_ids.len();
+        // 3. filter by version
+        let selected_object_number = self.extend_objects_to_delete_from_scan(&object_ids).await;
         metrics
             .full_gc_selected_object_count
-            .observe(selected_sst_number as _);
-        tracing::info!("GC watermark is {}. SST full scan returns {} SSTs. {} remains after filtered by GC watermark. {} remains after filtered by hummock version.",
-            watermark, candidate_sst_number, object_ids.len(), selected_sst_number);
-        Ok(selected_sst_number)
+            .observe(selected_object_number as _);
+        tracing::info!("GC watermark is {watermark}. Object full scan returns {candidate_object_number} objects. {after_watermark} remains after filtered by GC watermark. {after_time_travel} remains after filtered by time travel archives. {selected_object_number} remains after filtered by hummock version.");
+        Ok(selected_object_number)
     }
 }
 
@@ -334,17 +350,19 @@ mod tests {
 
         // No task scheduled because no available worker.
         assert!(!hummock_manager
-            .start_full_gc(Duration::from_secs(
-                hummock_manager.env.opts.min_sst_retention_time_sec - 1
-            ))
+            .start_full_gc(
+                Duration::from_secs(hummock_manager.env.opts.min_sst_retention_time_sec - 1,),
+                None
+            )
             .unwrap());
 
         let mut receiver = compactor_manager.add_compactor(context_id);
 
         assert!(hummock_manager
-            .start_full_gc(Duration::from_secs(
-                hummock_manager.env.opts.min_sst_retention_time_sec - 1
-            ))
+            .start_full_gc(
+                Duration::from_secs(hummock_manager.env.opts.min_sst_retention_time_sec - 1),
+                None
+            )
             .unwrap());
         let full_scan_task = match receiver.recv().await.unwrap().unwrap().event.unwrap() {
             ResponseEvent::FullScanTask(task) => task,
@@ -359,9 +377,10 @@ mod tests {
         );
 
         assert!(hummock_manager
-            .start_full_gc(Duration::from_secs(
-                hummock_manager.env.opts.min_sst_retention_time_sec + 1
-            ))
+            .start_full_gc(
+                Duration::from_secs(hummock_manager.env.opts.min_sst_retention_time_sec + 1),
+                None
+            )
             .unwrap());
         let full_scan_task = match receiver.recv().await.unwrap().unwrap().event.unwrap() {
             ResponseEvent::FullScanTask(task) => task,
@@ -411,7 +430,7 @@ mod tests {
         let committed_object_ids = sst_infos
             .into_iter()
             .flatten()
-            .map(|s| s.get_object_id())
+            .map(|s| s.object_id)
             .sorted()
             .collect_vec();
         assert!(!committed_object_ids.is_empty());
diff --git a/src/meta/src/hummock/manager/mod.rs b/src/meta/src/hummock/manager/mod.rs
index 65eede718c5c4..3d7fd1ee58ee8 100644
--- a/src/meta/src/hummock/manager/mod.rs
+++ b/src/meta/src/hummock/manager/mod.rs
@@ -20,7 +20,7 @@ use std::sync::Arc;
 use arc_swap::ArcSwap;
 use bytes::Bytes;
 use itertools::Itertools;
-use risingwave_common::monitor::rwlock::MonitoredRwLock;
+use risingwave_common::monitor::MonitoredRwLock;
 use risingwave_common::system_param::reader::SystemParamsRead;
 use risingwave_common::util::epoch::INVALID_EPOCH;
 use risingwave_hummock_sdk::version::{HummockVersion, HummockVersionDelta};
@@ -33,8 +33,8 @@ use risingwave_meta_model_v2::{
     hummock_version_delta, hummock_version_stats,
 };
 use risingwave_pb::hummock::{
-    CompactTaskAssignment, HummockPinnedSnapshot, HummockPinnedVersion, HummockSnapshot,
-    HummockVersionStats, PbCompactionGroupInfo, SubscribeCompactionEventRequest,
+    HummockPinnedSnapshot, HummockPinnedVersion, HummockSnapshot, HummockVersionStats,
+    PbCompactTaskAssignment, PbCompactionGroupInfo, SubscribeCompactionEventRequest,
 };
 use risingwave_pb::meta::subscribe_response::Operation;
 use tokio::sync::mpsc::UnboundedSender;
@@ -61,12 +61,15 @@ pub(crate) mod checkpoint;
 mod commit_epoch;
 mod compaction;
 pub mod sequence;
+pub mod time_travel;
 mod timer_task;
 mod transaction;
 mod utils;
 mod worker;
 
 pub(crate) use commit_epoch::*;
+#[cfg(any(test, feature = "test"))]
+pub use commit_epoch::{BatchCommitForNewCg, CommitEpochInfo};
 use compaction::*;
 pub use compaction::{check_cg_write_limit, WriteLimitType};
 pub(crate) use utils::*;
@@ -288,6 +291,7 @@ impl HummockManager {
             compaction_state: CompactionState::new(),
         };
         let instance = Arc::new(instance);
+        instance.init_time_travel_state().await?;
         instance.start_worker(rx).await;
         instance.load_meta_store_state().await?;
         instance.release_invalid_contexts().await?;
@@ -341,7 +345,7 @@ impl HummockManager {
         }
 
         compaction_guard.compact_task_assignment = match &meta_store {
-            MetaStoreImpl::Kv(meta_store) => CompactTaskAssignment::list(meta_store)
+            MetaStoreImpl::Kv(meta_store) => PbCompactTaskAssignment::list(meta_store)
                 .await?
                 .into_iter()
                 .map(|assigned| (assigned.key().unwrap(), assigned))
@@ -351,7 +355,12 @@ impl HummockManager {
                 .await
                 .map_err(MetadataModelError::from)?
                 .into_iter()
-                .map(|m| (m.id as HummockCompactionTaskId, m.into()))
+                .map(|m| {
+                    (
+                        m.id as HummockCompactionTaskId,
+                        PbCompactTaskAssignment::from(m),
+                    )
+                })
                 .collect(),
         };
 
@@ -463,8 +472,8 @@ impl HummockManager {
         };
 
         self.delete_object_tracker.clear();
-        // Not delete stale objects when archive is enabled
-        if !self.env.opts.enable_hummock_data_archive {
+        // Not delete stale objects when archive or time travel is enabled
+        if !self.env.opts.enable_hummock_data_archive && !self.time_travel_enabled().await {
             versioning_guard.mark_objects_for_deletion(context_info, &self.delete_object_tracker);
         }
 
diff --git a/src/meta/src/hummock/manager/tests.rs b/src/meta/src/hummock/manager/tests.rs
index c2abc138e4944..a0780641a3f30 100644
--- a/src/meta/src/hummock/manager/tests.rs
+++ b/src/meta/src/hummock/manager/tests.rs
@@ -24,20 +24,20 @@ use prometheus::Registry;
 use risingwave_common::catalog::TableId;
 use risingwave_common::util::epoch::{test_epoch, EpochExt, INVALID_EPOCH};
 use risingwave_hummock_sdk::compact::compact_task_to_string;
+use risingwave_hummock_sdk::compact_task::CompactTask;
 use risingwave_hummock_sdk::compaction_group::hummock_version_ext::get_compaction_group_ssts;
 use risingwave_hummock_sdk::compaction_group::StaticCompactionGroupId;
+use risingwave_hummock_sdk::key_range::KeyRange;
+use risingwave_hummock_sdk::sstable_info::SstableInfo;
 use risingwave_hummock_sdk::table_stats::{to_prost_table_stats_map, TableStats, TableStatsMap};
 use risingwave_hummock_sdk::version::HummockVersion;
 use risingwave_hummock_sdk::{
-    CompactionGroupId, ExtendedSstableInfo, HummockContextId, HummockEpoch, HummockSstableObjectId,
-    HummockVersionId, LocalSstableInfo, FIRST_VERSION_ID,
+    CompactionGroupId, HummockContextId, HummockEpoch, HummockSstableObjectId, HummockVersionId,
+    LocalSstableInfo, FIRST_VERSION_ID,
 };
 use risingwave_pb::common::{HostAddress, WorkerType};
 use risingwave_pb::hummock::compact_task::TaskStatus;
-use risingwave_pb::hummock::{
-    CompactTask, HummockPinnedSnapshot, HummockPinnedVersion, HummockSnapshot, KeyRange,
-    SstableInfo,
-};
+use risingwave_pb::hummock::{HummockPinnedSnapshot, HummockPinnedVersion, HummockSnapshot};
 use risingwave_pb::meta::add_worker_node_request::Property;
 
 use crate::hummock::compaction::compaction_config::CompactionConfigBuilder;
@@ -66,11 +66,11 @@ fn pin_snapshots_epoch(pin_snapshots: &[HummockPinnedSnapshot]) -> Vec<u64> {
 fn gen_sstable_info(sst_id: u64, idx: usize, table_ids: Vec<u32>) -> SstableInfo {
     SstableInfo {
         sst_id,
-        key_range: Some(KeyRange {
-            left: iterator_test_key_of_epoch(1, idx, 1),
-            right: iterator_test_key_of_epoch(1, idx, 1),
+        key_range: KeyRange {
+            left: iterator_test_key_of_epoch(1, idx, 1).into(),
+            right: iterator_test_key_of_epoch(1, idx, 1).into(),
             right_exclusive: false,
-        }),
+        },
         table_ids,
         object_id: sst_id,
         min_epoch: 20,
@@ -79,14 +79,8 @@ fn gen_sstable_info(sst_id: u64, idx: usize, table_ids: Vec<u32>) -> SstableInfo
     }
 }
 
-fn gen_extend_sstable_info(
-    sst_id: u64,
-    group_id: u64,
-    idx: usize,
-    table_ids: Vec<u32>,
-) -> ExtendedSstableInfo {
-    ExtendedSstableInfo {
-        compaction_group_id: group_id,
+fn gen_local_sstable_info(sst_id: u64, idx: usize, table_ids: Vec<u32>) -> LocalSstableInfo {
+    LocalSstableInfo {
         sst_info: gen_sstable_info(sst_id, idx, table_ids),
         table_stats: Default::default(),
     }
@@ -234,15 +228,8 @@ async fn test_hummock_compaction_task() {
         .await
         .unwrap()
         .unwrap();
-    assert_eq!(
-        compact_task
-            .get_input_ssts()
-            .first()
-            .unwrap()
-            .get_level_idx(),
-        0
-    );
-    assert_eq!(compact_task.get_task_id(), 2);
+    assert_eq!(compact_task.input_ssts.first().unwrap().level_idx, 0);
+    assert_eq!(compact_task.task_id, 2);
 
     // Cancel the task and succeed.
     assert!(hummock_manager
@@ -259,7 +246,7 @@ async fn test_hummock_compaction_task() {
         .await
         .unwrap()
         .unwrap();
-    assert_eq!(compact_task.get_task_id(), 3);
+    assert_eq!(compact_task.task_id, 3);
     // Finish the task and succeed.
 
     assert!(hummock_manager
@@ -295,15 +282,13 @@ async fn test_hummock_table() {
         Ordering::Equal,
         levels
             .l0
-            .as_ref()
-            .unwrap()
             .sub_levels
             .iter()
             .chain(levels.levels.iter())
             .flat_map(|level| level.table_infos.iter())
-            .map(|info| info.get_object_id())
+            .map(|info| info.object_id)
             .sorted()
-            .cmp(original_tables.iter().map(|ot| ot.get_object_id()).sorted())
+            .cmp(original_tables.iter().map(|ot| ot.object_id).sorted())
     );
 
     // Confirm tables got are equal to original tables
@@ -770,14 +755,7 @@ async fn test_print_compact_task() {
         .await
         .unwrap()
         .unwrap();
-    assert_eq!(
-        compact_task
-            .get_input_ssts()
-            .first()
-            .unwrap()
-            .get_level_idx(),
-        0
-    );
+    assert_eq!(compact_task.input_ssts.first().unwrap().level_idx, 0);
 
     let s = compact_task_to_string(&compact_task);
     assert!(s.contains("Compaction task id: 1, group-id: 2, type: Dynamic, target level: 0"));
@@ -799,7 +777,7 @@ async fn test_invalid_sst_id() {
     // reject due to invalid context id
     let sst_to_worker = ssts
         .iter()
-        .map(|LocalSstableInfo { sst_info, .. }| (sst_info.get_object_id(), WorkerId::MAX))
+        .map(|LocalSstableInfo { sst_info, .. }| (sst_info.object_id, WorkerId::MAX))
         .collect();
     let error = hummock_manager
         .commit_epoch_for_test(epoch, ssts.clone(), sst_to_worker)
@@ -809,7 +787,7 @@ async fn test_invalid_sst_id() {
 
     let sst_to_worker = ssts
         .iter()
-        .map(|LocalSstableInfo { sst_info, .. }| (sst_info.get_object_id(), context_id))
+        .map(|LocalSstableInfo { sst_info, .. }| (sst_info.object_id, context_id))
         .collect();
     hummock_manager
         .commit_epoch_for_test(epoch, ssts, sst_to_worker)
@@ -949,15 +927,8 @@ async fn test_hummock_compaction_task_heartbeat() {
         .unwrap()
         .unwrap();
 
-    assert_eq!(
-        compact_task
-            .get_input_ssts()
-            .first()
-            .unwrap()
-            .get_level_idx(),
-        0
-    );
-    assert_eq!(compact_task.get_task_id(), 2);
+    assert_eq!(compact_task.input_ssts.first().unwrap().level_idx, 0);
+    assert_eq!(compact_task.task_id, 2);
 
     for i in 0..10 {
         // send heartbeats to the task over 2.5 seconds
@@ -991,7 +962,7 @@ async fn test_hummock_compaction_task_heartbeat() {
         .unwrap()
         .unwrap();
 
-    assert_eq!(compact_task.get_task_id(), 3);
+    assert_eq!(compact_task.task_id, 3);
 
     // Cancel the task after heartbeat has triggered and fail.
 
@@ -1069,15 +1040,8 @@ async fn test_hummock_compaction_task_heartbeat_removal_on_node_removal() {
         .unwrap()
         .unwrap();
 
-    assert_eq!(
-        compact_task
-            .get_input_ssts()
-            .first()
-            .unwrap()
-            .get_level_idx(),
-        0
-    );
-    assert_eq!(compact_task.get_task_id(), 2);
+    assert_eq!(compact_task.input_ssts.first().unwrap().level_idx, 0);
+    assert_eq!(compact_task.task_id, 2);
 
     // send heartbeats to the task immediately
     let req = CompactTaskProgress {
@@ -1112,8 +1076,8 @@ async fn test_extend_objects_to_delete() {
         .iter()
         .map(|ssts| {
             ssts.iter()
-                .max_by_key(|s| s.get_object_id())
-                .map(|s| s.get_object_id())
+                .max_by_key(|s| s.object_id)
+                .map(|s| s.object_id)
                 .unwrap()
         })
         .max()
@@ -1122,7 +1086,7 @@ async fn test_extend_objects_to_delete() {
     let all_object_ids = sst_infos
         .iter()
         .flatten()
-        .map(|s| s.get_object_id())
+        .map(|s| s.object_id)
         .chain(max_committed_object_id + 1..=max_committed_object_id + orphan_sst_num)
         .collect_vec();
     assert!(hummock_manager.get_objects_to_delete().is_empty());
@@ -1188,7 +1152,7 @@ async fn test_extend_objects_to_delete() {
     hummock_manager
         .commit_epoch_for_test(
             new_epoch,
-            Vec::<ExtendedSstableInfo>::new(),
+            Vec::<LocalSstableInfo>::new(),
             Default::default(),
         )
         .await
@@ -1236,15 +1200,14 @@ async fn test_version_stats() {
         .into_iter()
         .enumerate()
         .map(|(idx, table_ids)| LocalSstableInfo {
-            compaction_group_id: StaticCompactionGroupId::StateDefault as _,
             sst_info: SstableInfo {
                 object_id: sst_ids[idx],
                 sst_id: sst_ids[idx],
-                key_range: Some(KeyRange {
-                    left: iterator_test_key_of_epoch(1, 1, 1),
-                    right: iterator_test_key_of_epoch(1, 1, 1),
+                key_range: KeyRange {
+                    left: iterator_test_key_of_epoch(1, 1, 1).into(),
+                    right: iterator_test_key_of_epoch(1, 1, 1).into(),
                     right_exclusive: false,
-                }),
+                },
                 file_size: 1024 * 1024 * 1024,
                 table_ids: table_ids.clone(),
                 ..Default::default()
@@ -1257,7 +1220,7 @@ async fn test_version_stats() {
         .collect_vec();
     let sst_to_worker = ssts
         .iter()
-        .map(|LocalSstableInfo { sst_info, .. }| (sst_info.get_object_id(), worker_node.id))
+        .map(|LocalSstableInfo { sst_info, .. }| (sst_info.object_id, worker_node.id))
         .collect();
     hummock_manager
         .commit_epoch_for_test(epoch, ssts, sst_to_worker)
@@ -1345,12 +1308,11 @@ async fn test_split_compaction_group_on_commit() {
         .register_table_ids_for_test(&[(101, 3)])
         .await
         .unwrap();
-    let sst_1 = ExtendedSstableInfo {
-        compaction_group_id: 2,
+    let sst_1 = LocalSstableInfo {
         sst_info: SstableInfo {
             object_id: 10,
             sst_id: 10,
-            key_range: None,
+            key_range: KeyRange::default(),
             table_ids: vec![100, 101],
             min_epoch: 20,
             max_epoch: 20,
@@ -1434,16 +1396,15 @@ async fn test_split_compaction_group_on_demand_basic() {
         .register_table_ids_for_test(&[(101, 2)])
         .await
         .unwrap();
-    let sst_1 = ExtendedSstableInfo {
-        compaction_group_id: 2,
+    let sst_1 = LocalSstableInfo {
         sst_info: SstableInfo {
             object_id: 10,
             sst_id: 10,
-            key_range: Some(KeyRange {
-                left: iterator_test_key_of_epoch(100, 1, 20),
-                right: iterator_test_key_of_epoch(100, 100, 20),
+            key_range: KeyRange {
+                left: iterator_test_key_of_epoch(100, 1, 20).into(),
+                right: iterator_test_key_of_epoch(100, 100, 20).into(),
                 right_exclusive: false,
-            }),
+            },
             table_ids: vec![100],
             min_epoch: 20,
             max_epoch: 20,
@@ -1451,16 +1412,15 @@ async fn test_split_compaction_group_on_demand_basic() {
         },
         table_stats: Default::default(),
     };
-    let sst_2 = ExtendedSstableInfo {
-        compaction_group_id: 2,
+    let sst_2 = LocalSstableInfo {
         sst_info: SstableInfo {
             object_id: 11,
             sst_id: 11,
-            key_range: Some(KeyRange {
-                left: iterator_test_key_of_epoch(100, 101, 20),
-                right: iterator_test_key_of_epoch(101, 100, 20),
+            key_range: KeyRange {
+                left: iterator_test_key_of_epoch(100, 101, 20).into(),
+                right: iterator_test_key_of_epoch(101, 100, 20).into(),
                 right_exclusive: false,
-            }),
+            },
             table_ids: vec![100, 101],
             min_epoch: 20,
             max_epoch: 20,
@@ -1534,12 +1494,11 @@ async fn test_split_compaction_group_on_demand_basic() {
 async fn test_split_compaction_group_on_demand_non_trivial() {
     let (_env, hummock_manager, _, worker_node) = setup_compute_env(80).await;
     let context_id = worker_node.id;
-    let sst_1 = ExtendedSstableInfo {
-        compaction_group_id: 2,
+    let sst_1 = LocalSstableInfo {
         sst_info: SstableInfo {
             object_id: 10,
             sst_id: 10,
-            key_range: None,
+            key_range: KeyRange::default(),
             table_ids: vec![100, 101],
             min_epoch: 20,
             max_epoch: 20,
@@ -1620,16 +1579,15 @@ async fn test_split_compaction_group_trivial_expired() {
         .register_table_ids_for_test(&[(101, 2)])
         .await
         .unwrap();
-    let sst_1 = ExtendedSstableInfo {
-        compaction_group_id: 2,
+    let sst_1 = LocalSstableInfo {
         sst_info: SstableInfo {
             object_id: 10,
             sst_id: 10,
-            key_range: Some(KeyRange {
-                left: iterator_test_key_of_epoch(100, 1, 20),
-                right: iterator_test_key_of_epoch(100, 100, 20),
+            key_range: KeyRange {
+                left: iterator_test_key_of_epoch(100, 1, 20).into(),
+                right: iterator_test_key_of_epoch(100, 100, 20).into(),
                 right_exclusive: false,
-            }),
+            },
             table_ids: vec![100],
             min_epoch: 20,
             max_epoch: 20,
@@ -1637,19 +1595,18 @@ async fn test_split_compaction_group_trivial_expired() {
         },
         table_stats: Default::default(),
     };
-    let sst_2 = ExtendedSstableInfo {
-        compaction_group_id: 2,
+    let sst_2 = LocalSstableInfo {
         sst_info: SstableInfo {
             object_id: 11,
             sst_id: 11,
             table_ids: vec![101],
             min_epoch: 20,
             max_epoch: 20,
-            key_range: Some(KeyRange {
-                left: iterator_test_key_of_epoch(101, 1, 20),
-                right: iterator_test_key_of_epoch(101, 100, 20),
+            key_range: KeyRange {
+                left: iterator_test_key_of_epoch(101, 1, 20).into(),
+                right: iterator_test_key_of_epoch(101, 100, 20).into(),
                 right_exclusive: false,
-            }),
+            },
             ..Default::default()
         },
         table_stats: Default::default(),
@@ -1737,7 +1694,7 @@ async fn test_split_compaction_group_trivial_expired() {
             vec![SstableInfo {
                 object_id: 12,
                 sst_id: 12,
-                key_range: None,
+                key_range: KeyRange::default(),
                 table_ids: vec![100],
                 min_epoch: 20,
                 max_epoch: 20,
@@ -1788,16 +1745,15 @@ async fn test_split_compaction_group_on_demand_bottom_levels() {
         .await
         .unwrap();
 
-    let sst_1 = ExtendedSstableInfo {
-        compaction_group_id: 2,
+    let sst_1 = LocalSstableInfo {
         sst_info: SstableInfo {
             object_id: 10,
             sst_id: 10,
-            key_range: Some(KeyRange {
-                left: iterator_test_key_of_epoch(1, 1, 1),
-                right: iterator_test_key_of_epoch(1, 1, 1),
+            key_range: KeyRange {
+                left: iterator_test_key_of_epoch(1, 1, 1).into(),
+                right: iterator_test_key_of_epoch(1, 1, 1).into(),
                 right_exclusive: false,
-            }),
+            },
             table_ids: vec![100, 101],
             min_epoch: 20,
             max_epoch: 20,
@@ -1824,22 +1780,22 @@ async fn test_split_compaction_group_on_demand_bottom_levels() {
                     object_id: 11,
                     sst_id: 11,
                     table_ids: vec![100, 101],
-                    key_range: Some(KeyRange {
-                        left: iterator_test_key_of_epoch(1, 1, 1),
-                        right: iterator_test_key_of_epoch(1, 1, 1),
+                    key_range: KeyRange {
+                        left: iterator_test_key_of_epoch(1, 1, 1).into(),
+                        right: iterator_test_key_of_epoch(1, 1, 1).into(),
                         right_exclusive: false,
-                    }),
+                    },
                     ..Default::default()
                 },
                 SstableInfo {
                     object_id: 12,
                     sst_id: 12,
                     table_ids: vec![100],
-                    key_range: Some(KeyRange {
-                        left: iterator_test_key_of_epoch(1, 2, 2),
-                        right: iterator_test_key_of_epoch(1, 2, 2),
+                    key_range: KeyRange {
+                        left: iterator_test_key_of_epoch(1, 2, 2).into(),
+                        right: iterator_test_key_of_epoch(1, 2, 2).into(),
                         right_exclusive: false,
-                    }),
+                    },
                     ..Default::default()
                 },
             ],
@@ -1851,8 +1807,6 @@ async fn test_split_compaction_group_on_demand_bottom_levels() {
     assert!(current_version
         .get_compaction_group_levels(2)
         .l0
-        .as_ref()
-        .unwrap()
         .sub_levels
         .is_empty());
     assert_eq!(
@@ -1878,7 +1832,7 @@ async fn test_split_compaction_group_on_demand_bottom_levels() {
     assert_eq!(
         current_version.get_compaction_group_levels(2).levels[base_level - 1].table_infos[0]
             .object_id,
-        sst_1.sst_info.get_object_id() + 1,
+        sst_1.sst_info.object_id + 1,
     );
     assert_eq!(
         current_version.get_compaction_group_levels(2).levels[base_level - 1].table_infos[0]
@@ -1924,16 +1878,15 @@ async fn test_compaction_task_expiration_due_to_split_group() {
         .register_table_ids_for_test(&[(101, 2)])
         .await
         .unwrap();
-    let sst_1 = ExtendedSstableInfo {
-        compaction_group_id: 2,
+    let sst_1 = LocalSstableInfo {
         sst_info: SstableInfo {
             object_id: 10,
             sst_id: 10,
-            key_range: Some(KeyRange {
-                left: iterator_test_key_of_epoch(1, 1, 1),
-                right: iterator_test_key_of_epoch(1, 1, 1),
+            key_range: KeyRange {
+                left: iterator_test_key_of_epoch(1, 1, 1).into(),
+                right: iterator_test_key_of_epoch(1, 1, 1).into(),
                 right_exclusive: false,
-            }),
+            },
             table_ids: vec![100, 101],
             min_epoch: 20,
             max_epoch: 20,
@@ -1941,16 +1894,15 @@ async fn test_compaction_task_expiration_due_to_split_group() {
         },
         table_stats: Default::default(),
     };
-    let sst_2 = ExtendedSstableInfo {
-        compaction_group_id: 2,
+    let sst_2 = LocalSstableInfo {
         sst_info: SstableInfo {
             object_id: 11,
             sst_id: 11,
-            key_range: Some(KeyRange {
-                left: iterator_test_key_of_epoch(1, 1, 1),
-                right: iterator_test_key_of_epoch(1, 1, 1),
+            key_range: KeyRange {
+                left: iterator_test_key_of_epoch(1, 1, 1).into(),
+                right: iterator_test_key_of_epoch(1, 1, 1).into(),
                 right_exclusive: false,
-            }),
+            },
             table_ids: vec![101],
             min_epoch: 20,
             max_epoch: 20,
@@ -2017,7 +1969,7 @@ async fn test_move_tables_between_compaction_group() {
         .register_table_ids_for_test(&[(102, 2)])
         .await
         .unwrap();
-    let sst_1 = gen_extend_sstable_info(10, 2, 1, vec![100, 101, 102]);
+    let sst_1 = gen_local_sstable_info(10, 1, vec![100, 101, 102]);
     hummock_manager
         .commit_epoch_for_test(30, vec![sst_1.clone()], HashMap::from([(10, context_id)]))
         .await
@@ -2040,7 +1992,7 @@ async fn test_move_tables_between_compaction_group() {
         )
         .await
         .unwrap());
-    let sst_2 = gen_extend_sstable_info(14, 2, 1, vec![101, 102]);
+    let sst_2 = gen_local_sstable_info(14, 1, vec![101, 102]);
     hummock_manager
         .commit_epoch_for_test(31, vec![sst_2.clone()], HashMap::from([(14, context_id)]))
         .await
@@ -2189,7 +2141,7 @@ async fn test_partition_level() {
         .register_table_ids_for_test(&[(101, 2)])
         .await
         .unwrap();
-    let sst_1 = gen_extend_sstable_info(10, 2, 1, vec![100, 101]);
+    let sst_1 = gen_local_sstable_info(10, 1, vec![100, 101]);
     hummock_manager
         .commit_epoch_for_test(30, vec![sst_1.clone()], HashMap::from([(10, context_id)]))
         .await
@@ -2226,11 +2178,12 @@ async fn test_partition_level() {
             .len(),
         1
     );
+
     let mut global_sst_id = 13;
     const MB: u64 = 1024 * 1024;
     let mut selector = default_compaction_selector();
     for epoch in 31..100 {
-        let mut sst = gen_extend_sstable_info(global_sst_id, new_group_id, 10, vec![100]);
+        let mut sst = gen_local_sstable_info(global_sst_id, 10, vec![100]);
         sst.sst_info.file_size = 10 * MB;
         sst.sst_info.uncompressed_file_size = 10 * MB;
         hummock_manager
@@ -2269,7 +2222,7 @@ async fn test_partition_level() {
     }
     let current_version = hummock_manager.get_current_version().await;
     let group = current_version.get_compaction_group_levels(new_group_id);
-    for sub_level in &group.l0.as_ref().unwrap().sub_levels {
+    for sub_level in &group.l0.sub_levels {
         if sub_level.total_file_size > config.sub_level_max_compaction_bytes {
             assert!(sub_level.vnode_partition_count > 0);
         }
@@ -2304,16 +2257,15 @@ async fn test_unregister_moved_table() {
         .register_table_ids_for_test(&[(101, 2)])
         .await
         .unwrap();
-    let sst_1 = ExtendedSstableInfo {
-        compaction_group_id: 2,
+    let sst_1 = LocalSstableInfo {
         sst_info: SstableInfo {
             object_id: 10,
             sst_id: 10,
-            key_range: Some(KeyRange {
-                left: iterator_test_key_of_epoch(100, 1, 20),
-                right: iterator_test_key_of_epoch(100, 100, 20),
+            key_range: KeyRange {
+                left: iterator_test_key_of_epoch(100, 1, 20).into(),
+                right: iterator_test_key_of_epoch(100, 100, 20).into(),
                 right_exclusive: false,
-            }),
+            },
             table_ids: vec![100],
             min_epoch: 20,
             max_epoch: 20,
@@ -2321,16 +2273,15 @@ async fn test_unregister_moved_table() {
         },
         table_stats: Default::default(),
     };
-    let sst_2 = ExtendedSstableInfo {
-        compaction_group_id: 2,
+    let sst_2 = LocalSstableInfo {
         sst_info: SstableInfo {
             object_id: 11,
             sst_id: 11,
-            key_range: Some(KeyRange {
-                left: iterator_test_key_of_epoch(100, 101, 20),
-                right: iterator_test_key_of_epoch(101, 100, 20),
+            key_range: KeyRange {
+                left: iterator_test_key_of_epoch(100, 101, 20).into(),
+                right: iterator_test_key_of_epoch(101, 100, 20).into(),
                 right_exclusive: false,
-            }),
+            },
             table_ids: vec![100, 101],
             min_epoch: 20,
             max_epoch: 20,
diff --git a/src/meta/src/hummock/manager/time_travel.rs b/src/meta/src/hummock/manager/time_travel.rs
new file mode 100644
index 0000000000000..e946969324b1a
--- /dev/null
+++ b/src/meta/src/hummock/manager/time_travel.rs
@@ -0,0 +1,515 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::{HashMap, HashSet, VecDeque};
+
+use anyhow::anyhow;
+use itertools::Itertools;
+use risingwave_common::system_param::reader::SystemParamsRead;
+use risingwave_common::util::epoch::Epoch;
+use risingwave_hummock_sdk::compaction_group::StaticCompactionGroupId;
+use risingwave_hummock_sdk::sstable_info::SstableInfo;
+use risingwave_hummock_sdk::time_travel::{
+    refill_version, IncompleteHummockVersion, IncompleteHummockVersionDelta,
+};
+use risingwave_hummock_sdk::version::{HummockVersion, HummockVersionDelta};
+use risingwave_hummock_sdk::{
+    CompactionGroupId, HummockEpoch, HummockSstableId, HummockSstableObjectId,
+};
+use risingwave_meta_model_v2::hummock_sstable_info::SstableInfoV2Backend;
+use risingwave_meta_model_v2::{
+    hummock_epoch_to_version, hummock_sstable_info, hummock_time_travel_delta,
+    hummock_time_travel_version,
+};
+use risingwave_pb::hummock::{PbHummockVersion, PbHummockVersionDelta};
+use sea_orm::sea_query::OnConflict;
+use sea_orm::ActiveValue::Set;
+use sea_orm::{
+    ColumnTrait, DatabaseTransaction, EntityTrait, QueryFilter, QueryOrder, QuerySelect,
+    TransactionTrait,
+};
+
+use crate::controller::SqlMetaStore;
+use crate::hummock::error::{Error, Result};
+use crate::hummock::HummockManager;
+use crate::manager::MetaStoreImpl;
+
+/// Time travel.
+impl HummockManager {
+    pub(crate) fn sql_store(&self) -> Option<SqlMetaStore> {
+        match self.env.meta_store() {
+            MetaStoreImpl::Sql(sql_store) => Some(sql_store),
+            _ => None,
+        }
+    }
+
+    pub(crate) async fn time_travel_enabled(&self) -> bool {
+        self.env
+            .system_params_reader()
+            .await
+            .time_travel_retention_ms()
+            > 0
+            && self.sql_store().is_some()
+    }
+
+    pub(crate) async fn init_time_travel_state(&self) -> Result<()> {
+        let Some(sql_store) = self.sql_store() else {
+            return Ok(());
+        };
+        let mut gurad = self.versioning.write().await;
+        gurad.mark_next_time_travel_version_snapshot();
+
+        gurad.last_time_travel_snapshot_sst_ids = HashSet::new();
+        let Some(version) = hummock_time_travel_version::Entity::find()
+            .order_by_desc(hummock_time_travel_version::Column::VersionId)
+            .one(&sql_store.conn)
+            .await?
+            .map(|v| HummockVersion::from_persisted_protobuf(&v.version.to_protobuf()))
+        else {
+            return Ok(());
+        };
+        gurad.last_time_travel_snapshot_sst_ids = version.get_sst_ids();
+        Ok(())
+    }
+
+    pub(crate) async fn truncate_time_travel_metadata(
+        &self,
+        epoch_watermark: HummockEpoch,
+    ) -> Result<()> {
+        let sql_store = match self.sql_store() {
+            Some(sql_store) => sql_store,
+            None => {
+                return Ok(());
+            }
+        };
+        let txn = sql_store.conn.begin().await?;
+
+        let version_watermark = hummock_epoch_to_version::Entity::find()
+            .filter(
+                hummock_epoch_to_version::Column::Epoch
+                    .lt(risingwave_meta_model_v2::Epoch::try_from(epoch_watermark).unwrap()),
+            )
+            .order_by_desc(hummock_epoch_to_version::Column::Epoch)
+            .one(&txn)
+            .await?;
+        let Some(version_watermark) = version_watermark else {
+            txn.commit().await?;
+            return Ok(());
+        };
+        let res = hummock_epoch_to_version::Entity::delete_many()
+            .filter(
+                hummock_epoch_to_version::Column::Epoch
+                    .lt(risingwave_meta_model_v2::Epoch::try_from(epoch_watermark).unwrap()),
+            )
+            .exec(&txn)
+            .await?;
+        tracing::debug!(
+            epoch_watermark,
+            "delete {} rows from hummock_epoch_to_version",
+            res.rows_affected
+        );
+        let earliest_valid_version = hummock_time_travel_version::Entity::find()
+            .filter(
+                hummock_time_travel_version::Column::VersionId.lte(version_watermark.version_id),
+            )
+            .order_by_desc(hummock_time_travel_version::Column::VersionId)
+            .one(&txn)
+            .await?
+            .map(|m| HummockVersion::from_persisted_protobuf(&m.version.to_protobuf()));
+        let Some(earliest_valid_version) = earliest_valid_version else {
+            txn.commit().await?;
+            return Ok(());
+        };
+        let (earliest_valid_version_id, earliest_valid_version_sst_ids) = {
+            (
+                earliest_valid_version.id,
+                earliest_valid_version.get_sst_ids(),
+            )
+        };
+        let version_ids_to_delete: Vec<risingwave_meta_model_v2::HummockVersionId> =
+            hummock_time_travel_version::Entity::find()
+                .select_only()
+                .column(hummock_time_travel_version::Column::VersionId)
+                .filter(
+                    hummock_time_travel_version::Column::VersionId.lt(earliest_valid_version_id),
+                )
+                .order_by_desc(hummock_time_travel_version::Column::VersionId)
+                .into_tuple()
+                .all(&txn)
+                .await?;
+        let delta_ids_to_delete: Vec<risingwave_meta_model_v2::HummockVersionId> =
+            hummock_time_travel_delta::Entity::find()
+                .select_only()
+                .column(hummock_time_travel_delta::Column::VersionId)
+                .filter(hummock_time_travel_delta::Column::VersionId.lt(earliest_valid_version_id))
+                .into_tuple()
+                .all(&txn)
+                .await?;
+        for delta_id_to_delete in delta_ids_to_delete {
+            let delta_to_delete = hummock_time_travel_delta::Entity::find_by_id(delta_id_to_delete)
+                .one(&txn)
+                .await?
+                .ok_or_else(|| {
+                    Error::TimeTravel(anyhow!(format!(
+                        "version delta {} not found",
+                        delta_id_to_delete
+                    )))
+                })?;
+            let new_sst_ids = HummockVersionDelta::from_persisted_protobuf(
+                &delta_to_delete.version_delta.to_protobuf(),
+            )
+            .newly_added_sst_ids();
+            // The SST ids added and then deleted by compaction between the 2 versions.
+            let sst_ids_to_delete = &new_sst_ids - &earliest_valid_version_sst_ids;
+            let res = hummock_sstable_info::Entity::delete_many()
+                .filter(hummock_sstable_info::Column::SstId.is_in(sst_ids_to_delete))
+                .exec(&txn)
+                .await?;
+            tracing::debug!(
+                delta_id = delta_to_delete.version_id,
+                "delete {} rows from hummock_sstable_info",
+                res.rows_affected
+            );
+        }
+        let mut next_version_sst_ids = earliest_valid_version_sst_ids;
+        for prev_version_id in version_ids_to_delete {
+            let sst_ids = {
+                let prev_version = hummock_time_travel_version::Entity::find_by_id(prev_version_id)
+                    .one(&txn)
+                    .await?
+                    .ok_or_else(|| {
+                        Error::TimeTravel(anyhow!(format!(
+                            "prev_version {} not found",
+                            prev_version_id
+                        )))
+                    })?;
+                HummockVersion::from_persisted_protobuf(&prev_version.version.to_protobuf())
+                    .get_sst_ids()
+            };
+            // The SST ids deleted by compaction between the 2 versions.
+            let sst_ids_to_delete = &sst_ids - &next_version_sst_ids;
+            let res = hummock_sstable_info::Entity::delete_many()
+                .filter(hummock_sstable_info::Column::SstId.is_in(sst_ids_to_delete))
+                .exec(&txn)
+                .await?;
+            tracing::debug!(
+                prev_version_id,
+                "delete {} rows from hummock_sstable_info",
+                res.rows_affected
+            );
+            next_version_sst_ids = sst_ids;
+        }
+
+        let res = hummock_time_travel_version::Entity::delete_many()
+            .filter(hummock_time_travel_version::Column::VersionId.lt(earliest_valid_version_id))
+            .exec(&txn)
+            .await?;
+        tracing::debug!(
+            epoch_watermark_version_id = version_watermark.version_id,
+            earliest_valid_version_id,
+            "delete {} rows from hummock_time_travel_version",
+            res.rows_affected
+        );
+
+        let res = hummock_time_travel_delta::Entity::delete_many()
+            .filter(hummock_time_travel_delta::Column::VersionId.lt(earliest_valid_version_id))
+            .exec(&txn)
+            .await?;
+        tracing::debug!(
+            epoch_watermark_version_id = version_watermark.version_id,
+            earliest_valid_version_id,
+            "delete {} rows from hummock_time_travel_delta",
+            res.rows_affected
+        );
+
+        txn.commit().await?;
+        Ok(())
+    }
+
+    pub(crate) async fn all_object_ids_in_time_travel(
+        &self,
+    ) -> Result<impl Iterator<Item = HummockSstableId>> {
+        let object_ids: Vec<risingwave_meta_model_v2::HummockSstableObjectId> =
+            match self.sql_store() {
+                Some(sql_store) => {
+                    hummock_sstable_info::Entity::find()
+                        .select_only()
+                        .column(hummock_sstable_info::Column::ObjectId)
+                        .into_tuple()
+                        .all(&sql_store.conn)
+                        .await?
+                }
+                None => {
+                    vec![]
+                }
+            };
+        let object_ids = object_ids
+            .into_iter()
+            .unique()
+            .map(|object_id| HummockSstableObjectId::try_from(object_id).unwrap());
+        Ok(object_ids)
+    }
+
+    /// Attempt to locate the version corresponding to `query_epoch`.
+    ///
+    /// The version is retrieved from `hummock_epoch_to_version`, selecting the entry with the largest epoch that's lte `query_epoch`.
+    ///
+    /// The resulted version is complete, i.e. with correct `SstableInfo`.
+    pub async fn epoch_to_version(&self, query_epoch: HummockEpoch) -> Result<HummockVersion> {
+        let sql_store = self.sql_store().ok_or_else(require_sql_meta_store_err)?;
+        let epoch_to_version = hummock_epoch_to_version::Entity::find()
+            .filter(
+                hummock_epoch_to_version::Column::Epoch
+                    .lte(risingwave_meta_model_v2::Epoch::try_from(query_epoch).unwrap()),
+            )
+            .order_by_desc(hummock_epoch_to_version::Column::Epoch)
+            .one(&sql_store.conn)
+            .await?
+            .ok_or_else(|| {
+                Error::TimeTravel(anyhow!(format!(
+                    "version not found for epoch {}",
+                    query_epoch
+                )))
+            })?;
+        let actual_version_id = epoch_to_version.version_id;
+        tracing::debug!(
+            query_epoch,
+            query_tz = ?(Epoch(query_epoch).as_timestamptz()),
+            actual_epoch = epoch_to_version.epoch,
+            actual_tz = ?(Epoch(u64::try_from(epoch_to_version.epoch).unwrap()).as_timestamptz()),
+            actual_version_id,
+            "convert query epoch"
+        );
+
+        let replay_version = hummock_time_travel_version::Entity::find()
+            .filter(hummock_time_travel_version::Column::VersionId.lte(actual_version_id))
+            .order_by_desc(hummock_time_travel_version::Column::VersionId)
+            .one(&sql_store.conn)
+            .await?
+            .ok_or_else(|| {
+                Error::TimeTravel(anyhow!(format!(
+                    "no replay version found for epoch {}, version {}",
+                    query_epoch, actual_version_id,
+                )))
+            })?;
+        let deltas = hummock_time_travel_delta::Entity::find()
+            .filter(hummock_time_travel_delta::Column::VersionId.gt(replay_version.version_id))
+            .filter(hummock_time_travel_delta::Column::VersionId.lte(actual_version_id))
+            .order_by_asc(hummock_time_travel_delta::Column::VersionId)
+            .all(&sql_store.conn)
+            .await?;
+        let mut actual_version = replay_archive(
+            replay_version.version.to_protobuf(),
+            deltas.into_iter().map(|d| d.version_delta.to_protobuf()),
+        );
+
+        let mut sst_ids = actual_version
+            .get_sst_ids()
+            .into_iter()
+            .collect::<VecDeque<_>>();
+        let sst_count = sst_ids.len();
+        let mut sst_id_to_info = HashMap::with_capacity(sst_count);
+        let sst_info_fetch_batch_size = std::env::var("RW_TIME_TRAVEL_SST_INFO_FETCH_BATCH_SIZE")
+            .unwrap_or_else(|_| "100".into())
+            .parse()
+            .unwrap();
+        while !sst_ids.is_empty() {
+            let sst_infos = hummock_sstable_info::Entity::find()
+                .filter(hummock_sstable_info::Column::SstId.is_in(
+                    sst_ids.drain(..std::cmp::min(sst_info_fetch_batch_size, sst_ids.len())),
+                ))
+                .all(&sql_store.conn)
+                .await?;
+            for sst_info in sst_infos {
+                let sst_info: SstableInfo = sst_info.sstable_info.to_protobuf().into();
+                sst_id_to_info.insert(sst_info.sst_id, sst_info);
+            }
+        }
+        if sst_count != sst_id_to_info.len() {
+            return Err(Error::TimeTravel(anyhow!(format!(
+                "some SstableInfos not found for epoch {}, version {}",
+                query_epoch, actual_version_id,
+            ))));
+        }
+        refill_version(&mut actual_version, &sst_id_to_info);
+        Ok(actual_version)
+    }
+
+    pub(crate) async fn write_time_travel_metadata(
+        &self,
+        txn: &DatabaseTransaction,
+        version: Option<&HummockVersion>,
+        delta: HummockVersionDelta,
+        group_parents: &HashMap<CompactionGroupId, CompactionGroupId>,
+        skip_sst_ids: &HashSet<HummockSstableId>,
+    ) -> Result<Option<HashSet<HummockSstableId>>> {
+        async fn write_sstable_infos(
+            sst_infos: impl Iterator<Item = &SstableInfo>,
+            txn: &DatabaseTransaction,
+        ) -> Result<usize> {
+            let mut count = 0;
+            for sst_info in sst_infos {
+                let m = hummock_sstable_info::ActiveModel {
+                    sst_id: Set(sst_info.sst_id.try_into().unwrap()),
+                    object_id: Set(sst_info.object_id.try_into().unwrap()),
+                    sstable_info: Set(SstableInfoV2Backend::from(&sst_info.to_protobuf())),
+                };
+                hummock_sstable_info::Entity::insert(m)
+                    .on_conflict(
+                        OnConflict::column(hummock_sstable_info::Column::SstId)
+                            .do_nothing()
+                            .to_owned(),
+                    )
+                    .do_nothing()
+                    .exec(txn)
+                    .await?;
+                count += 1;
+            }
+            Ok(count)
+        }
+
+        let epoch = delta.max_committed_epoch;
+        let version_id = delta.id;
+        let m = hummock_epoch_to_version::ActiveModel {
+            epoch: Set(epoch.try_into().unwrap()),
+            version_id: Set(version_id.try_into().unwrap()),
+        };
+        hummock_epoch_to_version::Entity::insert(m)
+            .exec(txn)
+            .await?;
+
+        let mut version_sst_ids = None;
+        let select_groups = group_parents
+            .iter()
+            .filter_map(|(cg_id, _)| {
+                if should_ignore_group(find_root_group(*cg_id, group_parents)) {
+                    None
+                } else {
+                    Some(*cg_id)
+                }
+            })
+            .collect::<HashSet<_>>();
+        if let Some(version) = version {
+            version_sst_ids = Some(
+                version
+                    .get_sst_infos_from_groups(&select_groups)
+                    .map(|s| s.sst_id)
+                    .collect(),
+            );
+            write_sstable_infos(
+                version
+                    .get_sst_infos_from_groups(&select_groups)
+                    .filter(|s| !skip_sst_ids.contains(&s.sst_id)),
+                txn,
+            )
+            .await?;
+            let m = hummock_time_travel_version::ActiveModel {
+                version_id: Set(
+                    risingwave_meta_model_v2::HummockVersionId::try_from(version.id).unwrap(),
+                ),
+                version: Set((&IncompleteHummockVersion::from((version, &select_groups))
+                    .to_protobuf())
+                    .into()),
+            };
+            hummock_time_travel_version::Entity::insert(m)
+                .on_conflict(
+                    OnConflict::column(hummock_time_travel_version::Column::VersionId)
+                        .do_nothing()
+                        .to_owned(),
+                )
+                .do_nothing()
+                .exec(txn)
+                .await?;
+        }
+        let written = write_sstable_infos(
+            delta
+                .newly_added_sst_infos(&select_groups)
+                .filter(|s| !skip_sst_ids.contains(&s.sst_id)),
+            txn,
+        )
+        .await?;
+        // Ignore delta which adds no data.
+        if written > 0 {
+            let m = hummock_time_travel_delta::ActiveModel {
+                version_id: Set(
+                    risingwave_meta_model_v2::HummockVersionId::try_from(delta.id).unwrap(),
+                ),
+                version_delta: Set((&IncompleteHummockVersionDelta::from((
+                    &delta,
+                    &select_groups,
+                ))
+                .to_protobuf())
+                    .into()),
+            };
+            hummock_time_travel_delta::Entity::insert(m)
+                .on_conflict(
+                    OnConflict::column(hummock_time_travel_delta::Column::VersionId)
+                        .do_nothing()
+                        .to_owned(),
+                )
+                .do_nothing()
+                .exec(txn)
+                .await?;
+        }
+
+        Ok(version_sst_ids)
+    }
+}
+
+fn replay_archive(
+    version: PbHummockVersion,
+    deltas: impl Iterator<Item = PbHummockVersionDelta>,
+) -> HummockVersion {
+    let mut last_version = HummockVersion::from_persisted_protobuf(&version);
+    let mut mce = last_version.max_committed_epoch;
+    for d in deltas {
+        let d = HummockVersionDelta::from_persisted_protobuf(&d);
+        assert!(
+            d.max_committed_epoch > mce,
+            "time travel expects delta from commit_epoch only"
+        );
+        mce = d.max_committed_epoch;
+        // Need to work around the assertion in `apply_version_delta`.
+        // Because compaction deltas are not included in time travel archive.
+        while last_version.id < d.prev_id {
+            last_version.id += 1;
+        }
+        last_version.apply_version_delta(&d);
+    }
+    last_version
+}
+
+fn find_root_group(
+    group_id: CompactionGroupId,
+    parents: &HashMap<CompactionGroupId, CompactionGroupId>,
+) -> CompactionGroupId {
+    let mut root = group_id;
+    while let Some(parent) = parents.get(&root)
+        && *parent != 0
+    {
+        root = *parent;
+    }
+    root
+}
+
+fn should_ignore_group(root_group_id: CompactionGroupId) -> bool {
+    // It is possible some intermediate groups has been dropped,
+    // so it's impossible to tell whether the root group is MaterializedView or not.
+    // Just treat them as MaterializedView for correctness.
+    root_group_id == StaticCompactionGroupId::StateDefault as CompactionGroupId
+}
+
+pub fn require_sql_meta_store_err() -> Error {
+    Error::TimeTravel(anyhow!("require SQL meta store"))
+}
diff --git a/src/meta/src/hummock/manager/timer_task.rs b/src/meta/src/hummock/manager/timer_task.rs
index bb4a9fa86b06c..ec0f77ac88a8a 100644
--- a/src/meta/src/hummock/manager/timer_task.rs
+++ b/src/meta/src/hummock/manager/timer_task.rs
@@ -305,6 +305,13 @@ impl HummockManager {
                                             compact_task::TaskType::SpaceReclaim,
                                         )
                                         .await;
+
+                                    // share the same trigger with SpaceReclaim
+                                    hummock_manager
+                                        .on_handle_trigger_multi_group(
+                                            compact_task::TaskType::VnodeWatermark,
+                                        )
+                                        .await;
                                 }
 
                                 HummockTimerEvent::TtlCompactionTrigger => {
@@ -333,7 +340,7 @@ impl HummockManager {
 
                                 HummockTimerEvent::FullGc => {
                                     if hummock_manager
-                                        .start_full_gc(Duration::from_secs(3600))
+                                        .start_full_gc(Duration::from_secs(3600), None)
                                         .is_ok()
                                     {
                                         tracing::info!("Start full GC from meta node.");
@@ -369,8 +376,6 @@ impl HummockManager {
                         *id,
                         group
                             .l0
-                            .as_ref()
-                            .unwrap()
                             .sub_levels
                             .iter()
                             .map(|level| level.total_file_size)
diff --git a/src/meta/src/hummock/manager/transaction.rs b/src/meta/src/hummock/manager/transaction.rs
index c467e95adfdbe..9833dbea6e3af 100644
--- a/src/meta/src/hummock/manager/transaction.rs
+++ b/src/meta/src/hummock/manager/transaction.rs
@@ -12,12 +12,22 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::BTreeMap;
+use std::collections::{BTreeMap, HashMap};
 use std::ops::{Deref, DerefMut};
 
-use risingwave_hummock_sdk::version::{HummockVersion, HummockVersionDelta};
-use risingwave_hummock_sdk::HummockVersionId;
-use risingwave_pb::hummock::HummockVersionStats;
+use risingwave_common::catalog::TableId;
+use risingwave_hummock_sdk::change_log::ChangeLogDelta;
+use risingwave_hummock_sdk::compaction_group::StaticCompactionGroupId;
+use risingwave_hummock_sdk::sstable_info::SstableInfo;
+use risingwave_hummock_sdk::table_watermark::TableWatermarks;
+use risingwave_hummock_sdk::version::{
+    GroupDelta, HummockVersion, HummockVersionDelta, IntraLevelDelta,
+};
+use risingwave_hummock_sdk::{CompactionGroupId, HummockEpoch, HummockVersionId, LocalSstableInfo};
+use risingwave_pb::hummock::{
+    CompactionConfig, CompatibilityVersion, GroupConstruct, HummockVersionStats,
+    StateTableInfoDelta,
+};
 use risingwave_pb::meta::subscribe_response::{Info, Operation};
 
 use crate::manager::NotificationManager;
@@ -101,6 +111,116 @@ impl<'a> HummockVersionTransaction<'a> {
         version.apply_version_delta(&delta);
         deltas.push(delta);
     }
+
+    /// Returns a duplicate delta, used by time travel.
+    #[expect(clippy::type_complexity)]
+    pub(super) fn pre_commit_epoch(
+        &mut self,
+        max_committed_epoch: HummockEpoch,
+        commit_sstables: BTreeMap<CompactionGroupId, Vec<SstableInfo>>,
+        new_table_ids: HashMap<TableId, CompactionGroupId>,
+        new_table_watermarks: HashMap<TableId, TableWatermarks>,
+        change_log_delta: HashMap<TableId, ChangeLogDelta>,
+        batch_commit_for_new_cg: Option<(
+            HashMap<CompactionGroupId, BTreeMap<u64, Vec<LocalSstableInfo>>>,
+            CompactionConfig,
+        )>,
+    ) -> HummockVersionDelta {
+        let mut new_version_delta = self.new_delta();
+        new_version_delta.max_committed_epoch = max_committed_epoch;
+        new_version_delta.new_table_watermarks = new_table_watermarks;
+        new_version_delta.change_log_delta = change_log_delta;
+
+        if let Some((batch_commit_for_new_cg, compaction_group_config)) = batch_commit_for_new_cg {
+            for (compaction_group_id, batch_commit_sst) in batch_commit_for_new_cg {
+                let group_deltas = &mut new_version_delta
+                    .group_deltas
+                    .entry(compaction_group_id)
+                    .or_default()
+                    .group_deltas;
+
+                #[expect(deprecated)]
+                group_deltas.push(GroupDelta::GroupConstruct(GroupConstruct {
+                    group_config: Some(compaction_group_config.clone()),
+                    group_id: compaction_group_id,
+                    parent_group_id: StaticCompactionGroupId::NewCompactionGroup
+                        as CompactionGroupId,
+                    new_sst_start_id: 0, // No need to set it when `NewCompactionGroup`
+                    table_ids: vec![],
+                    version: CompatibilityVersion::NoMemberTableIds as i32,
+                }));
+
+                for (epoch, insert_ssts) in batch_commit_sst {
+                    assert!(epoch < max_committed_epoch);
+                    let l0_sub_level_id = epoch;
+                    let group_delta = GroupDelta::IntraLevel(IntraLevelDelta::new(
+                        0,
+                        l0_sub_level_id,
+                        vec![], // default
+                        insert_ssts.into_iter().map(|s| s.sst_info).collect(),
+                        0, // default
+                    ));
+                    group_deltas.push(group_delta);
+                }
+            }
+        }
+
+        // Append SSTs to a new version.
+        for (compaction_group_id, inserted_table_infos) in commit_sstables {
+            let group_deltas = &mut new_version_delta
+                .group_deltas
+                .entry(compaction_group_id)
+                .or_default()
+                .group_deltas;
+            let l0_sub_level_id = max_committed_epoch;
+            let group_delta = GroupDelta::IntraLevel(IntraLevelDelta::new(
+                0,
+                l0_sub_level_id,
+                vec![], // default
+                inserted_table_infos,
+                0, // default
+            ));
+
+            group_deltas.push(group_delta);
+        }
+
+        // update state table info
+        new_version_delta.with_latest_version(|version, delta| {
+            for (table_id, cg_id) in new_table_ids {
+                assert!(
+                    !version.state_table_info.info().contains_key(&table_id),
+                    "newly added table exists previously: {:?}",
+                    table_id
+                );
+                delta.state_table_info_delta.insert(
+                    table_id,
+                    StateTableInfoDelta {
+                        committed_epoch: max_committed_epoch,
+                        safe_epoch: max_committed_epoch,
+                        compaction_group_id: cg_id,
+                    },
+                );
+            }
+
+            for (table_id, info) in version.state_table_info.info() {
+                assert!(delta
+                    .state_table_info_delta
+                    .insert(
+                        *table_id,
+                        StateTableInfoDelta {
+                            committed_epoch: max_committed_epoch,
+                            safe_epoch: info.safe_epoch,
+                            compaction_group_id: info.compaction_group_id,
+                        }
+                    )
+                    .is_none(),);
+            }
+        });
+
+        let time_travel_delta = (*new_version_delta).clone();
+        new_version_delta.pre_apply();
+        time_travel_delta
+    }
 }
 
 impl<'a> InMemValTransaction for HummockVersionTransaction<'a> {
diff --git a/src/meta/src/hummock/manager/utils.rs b/src/meta/src/hummock/manager/utils.rs
index 3d8cb04546284..fd1372b3a5009 100644
--- a/src/meta/src/hummock/manager/utils.rs
+++ b/src/meta/src/hummock/manager/utils.rs
@@ -24,11 +24,11 @@ macro_rules! commit_multi_var {
                     $crate::manager::MetaStoreImpl::Kv(meta_store) => {
                         use crate::storage::Transaction;
                         use crate::storage::meta_store::MetaStore;
-                        let mut trx = Transaction::default();
+                        let mut txn = Transaction::default();
                         $(
-                            $val_txn.apply_to_txn(&mut trx).await?;
+                            $val_txn.apply_to_txn(&mut txn).await?;
                         )*
-                        meta_store.txn(trx).await?;
+                        meta_store.txn(txn).await?;
                         $(
                             $val_txn.commit();
                         )*
@@ -37,11 +37,11 @@ macro_rules! commit_multi_var {
                     crate::manager::MetaStoreImpl::Sql(sql_meta_store) => {
                         use sea_orm::TransactionTrait;
                         use crate::model::MetadataModelError;
-                        let mut trx = sql_meta_store.conn.begin().await.map_err(MetadataModelError::from)?;
+                        let mut txn = sql_meta_store.conn.begin().await.map_err(MetadataModelError::from)?;
                         $(
-                            $val_txn.apply_to_txn(&mut trx).await?;
+                            $val_txn.apply_to_txn(&mut txn).await?;
                         )*
-                        trx.commit().await.map_err(MetadataModelError::from)?;
+                        txn.commit().await.map_err(MetadataModelError::from)?;
                         $(
                             $val_txn.commit();
                         )*
@@ -52,8 +52,28 @@ macro_rules! commit_multi_var {
         }
     };
 }
-pub(crate) use commit_multi_var;
+
+macro_rules! commit_multi_var_with_provided_txn {
+    ($txn:expr, $($val_txn:expr),*) => {
+        {
+            async {
+                use crate::model::{InMemValTransaction, ValTransaction};
+                use crate::model::MetadataModelError;
+                $(
+                    $val_txn.apply_to_txn(&mut $txn).await?;
+                )*
+                $txn.commit().await.map_err(MetadataModelError::from)?;
+                $(
+                    $val_txn.commit();
+                )*
+                Result::Ok(())
+            }.await
+        }
+    };
+}
+
 use risingwave_hummock_sdk::SstObjectIdRange;
+pub(crate) use {commit_multi_var, commit_multi_var_with_provided_txn};
 
 use crate::hummock::error::Result;
 use crate::hummock::sequence::next_sstable_object_id;
diff --git a/src/meta/src/hummock/manager/versioning.rs b/src/meta/src/hummock/manager/versioning.rs
index 2e6b2512a8be0..1bea45720c1b2 100644
--- a/src/meta/src/hummock/manager/versioning.rs
+++ b/src/meta/src/hummock/manager/versioning.rs
@@ -21,16 +21,17 @@ use risingwave_hummock_sdk::compaction_group::hummock_version_ext::{
     get_compaction_group_ids, get_table_compaction_group_id_mapping, BranchedSstInfo,
 };
 use risingwave_hummock_sdk::compaction_group::StateTableId;
+use risingwave_hummock_sdk::sstable_info::SstableInfo;
 use risingwave_hummock_sdk::table_stats::add_prost_table_stats_map;
 use risingwave_hummock_sdk::version::{HummockVersion, HummockVersionDelta};
 use risingwave_hummock_sdk::{
-    CompactionGroupId, HummockContextId, HummockEpoch, HummockSstableObjectId, HummockVersionId,
+    CompactionGroupId, HummockContextId, HummockEpoch, HummockSstableId, HummockSstableObjectId,
+    HummockVersionId,
 };
 use risingwave_pb::common::WorkerNode;
 use risingwave_pb::hummock::write_limits::WriteLimit;
 use risingwave_pb::hummock::{
-    HummockPinnedSnapshot, HummockPinnedVersion, HummockSnapshot, HummockVersionStats, SstableInfo,
-    TableStats,
+    HummockPinnedSnapshot, HummockPinnedVersion, HummockSnapshot, HummockVersionStats, TableStats,
 };
 use risingwave_pb::meta::subscribe_response::{Info, Operation};
 
@@ -56,6 +57,11 @@ pub struct Versioning {
     /// Latest hummock version
     pub current_version: HummockVersion,
     pub local_metrics: HashMap<u32, LocalTableMetrics>,
+    pub time_travel_snapshot_interval_counter: u64,
+    /// Used to avoid the attempts to rewrite the same SST to meta store
+    pub last_time_travel_snapshot_sst_ids: HashSet<HummockSstableId>,
+    /// Whether time travel is enabled during last commit epoch.
+    pub time_travel_toggle_check: bool,
 
     // Persistent states below
     pub hummock_version_deltas: BTreeMap<HummockVersionId, HummockVersionDelta>,
@@ -95,6 +101,10 @@ impl Versioning {
                 .flat_map(|(_, stale_objects)| stale_objects.id.iter().cloned()),
         );
     }
+
+    pub(super) fn mark_next_time_travel_version_snapshot(&mut self) {
+        self.time_travel_snapshot_interval_counter = u64::MAX;
+    }
 }
 
 impl HummockManager {
@@ -373,7 +383,7 @@ fn estimate_table_stats(sst: &SstableInfo) -> HashMap<u32, TableStats> {
     let mut changes: HashMap<u32, TableStats> = HashMap::default();
     let weighted_value =
         |value: i64| -> i64 { (value as f64 / sst.table_ids.len() as f64).ceil() as i64 };
-    let key_range = sst.key_range.as_ref().unwrap();
+    let key_range = &sst.key_range;
     let estimated_key_size: u64 = (key_range.left.len() + key_range.right.len()) as u64 / 2;
     let mut estimated_total_key_size = estimated_key_size * sst.total_key_count;
     if estimated_total_key_size > sst.uncompressed_file_size {
@@ -395,13 +405,13 @@ mod tests {
     use std::collections::HashMap;
     use std::sync::Arc;
 
+    use risingwave_hummock_sdk::key_range::KeyRange;
+    use risingwave_hummock_sdk::level::{Level, Levels};
+    use risingwave_hummock_sdk::sstable_info::SstableInfo;
     use risingwave_hummock_sdk::version::HummockVersion;
     use risingwave_hummock_sdk::{CompactionGroupId, HummockVersionId};
-    use risingwave_pb::hummock::hummock_version::Levels;
     use risingwave_pb::hummock::write_limits::WriteLimit;
-    use risingwave_pb::hummock::{
-        HummockPinnedVersion, HummockVersionStats, KeyRange, Level, OverlappingLevel, SstableInfo,
-    };
+    use risingwave_pb::hummock::{HummockPinnedVersion, HummockVersionStats};
 
     use crate::hummock::compaction::compaction_config::CompactionConfigBuilder;
     use crate::hummock::manager::context::ContextInfo;
@@ -433,12 +443,7 @@ mod tests {
     #[test]
     fn test_calc_new_write_limits() {
         let add_level_to_l0 = |levels: &mut Levels| {
-            levels
-                .l0
-                .as_mut()
-                .unwrap()
-                .sub_levels
-                .push(Level::default());
+            levels.l0.sub_levels.push(Level::default());
         };
         let set_sub_level_number_threshold_for_group_1 =
             |target_groups: &mut HashMap<CompactionGroupId, CompactionGroup>,
@@ -471,13 +476,7 @@ mod tests {
         .collect();
         let mut version: HummockVersion = Default::default();
         for group_id in 1..=3 {
-            version.levels.insert(
-                group_id,
-                Levels {
-                    l0: Some(OverlappingLevel::default()),
-                    ..Default::default()
-                },
-            );
+            version.levels.insert(group_id, Levels::default());
         }
         let new_write_limits =
             calc_new_write_limits(target_groups.clone(), origin_snapshot.clone(), &version);
@@ -532,11 +531,11 @@ mod tests {
     #[test]
     fn test_estimate_table_stats() {
         let sst = SstableInfo {
-            key_range: Some(KeyRange {
-                left: vec![1; 10],
-                right: vec![1; 20],
+            key_range: KeyRange {
+                left: vec![1; 10].into(),
+                right: vec![1; 20].into(),
                 ..Default::default()
-            }),
+            },
             table_ids: vec![1, 2, 3],
             total_key_count: 6000,
             uncompressed_file_size: 6_000_000,
@@ -564,7 +563,6 @@ mod tests {
                         table_infos: vec![sst.clone()],
                         ..Default::default()
                     }],
-                    l0: Some(Default::default()),
                     ..Default::default()
                 },
             );
@@ -594,11 +592,11 @@ mod tests {
     #[test]
     fn test_estimate_table_stats_large_key_range() {
         let sst = SstableInfo {
-            key_range: Some(KeyRange {
-                left: vec![1; 1000],
-                right: vec![1; 2000],
+            key_range: KeyRange {
+                left: vec![1; 1000].into(),
+                right: vec![1; 2000].into(),
                 ..Default::default()
-            }),
+            },
             table_ids: vec![1, 2, 3],
             total_key_count: 6000,
             uncompressed_file_size: 60_000,
diff --git a/src/meta/src/hummock/metrics_utils.rs b/src/meta/src/hummock/metrics_utils.rs
index 3779ff5b2be97..0b23a6970965c 100644
--- a/src/meta/src/hummock/metrics_utils.rs
+++ b/src/meta/src/hummock/metrics_utils.rs
@@ -21,10 +21,10 @@ use itertools::{enumerate, Itertools};
 use prometheus::core::{AtomicU64, GenericCounter};
 use prometheus::IntGauge;
 use risingwave_hummock_sdk::compaction_group::hummock_version_ext::object_size_map;
+use risingwave_hummock_sdk::level::Levels;
 use risingwave_hummock_sdk::table_stats::PbTableStatsMap;
 use risingwave_hummock_sdk::version::HummockVersion;
 use risingwave_hummock_sdk::{CompactionGroupId, HummockContextId, HummockEpoch, HummockVersionId};
-use risingwave_pb::hummock::hummock_version::Levels;
 use risingwave_pb::hummock::write_limits::WriteLimit;
 use risingwave_pb::hummock::{
     CompactionConfig, HummockPinnedSnapshot, HummockPinnedVersion, HummockVersionStats, LevelType,
@@ -224,42 +224,42 @@ pub fn trigger_sst_stat(
         let overlapping_sst_num = current_version
             .levels
             .get(&compaction_group_id)
-            .and_then(|level| {
-                level.l0.as_ref().map(|l0| {
-                    l0.sub_levels
-                        .iter()
-                        .filter(|sub_level| sub_level.level_type() == LevelType::Overlapping)
-                        .count()
-                })
+            .map(|level| {
+                level
+                    .l0
+                    .sub_levels
+                    .iter()
+                    .filter(|sub_level| sub_level.level_type == LevelType::Overlapping)
+                    .count()
             })
             .unwrap_or(0);
 
         let non_overlap_sst_num = current_version
             .levels
             .get(&compaction_group_id)
-            .and_then(|level| {
-                level.l0.as_ref().map(|l0| {
-                    l0.sub_levels
-                        .iter()
-                        .filter(|sub_level| sub_level.level_type() == LevelType::Nonoverlapping)
-                        .count()
-                })
+            .map(|level| {
+                level
+                    .l0
+                    .sub_levels
+                    .iter()
+                    .filter(|sub_level| sub_level.level_type == LevelType::Nonoverlapping)
+                    .count()
             })
             .unwrap_or(0);
 
         let partition_level_num = current_version
             .levels
             .get(&compaction_group_id)
-            .and_then(|level| {
-                level.l0.as_ref().map(|l0| {
-                    l0.sub_levels
-                        .iter()
-                        .filter(|sub_level| {
-                            sub_level.level_type() == LevelType::Nonoverlapping
-                                && sub_level.vnode_partition_count > 0
-                        })
-                        .count()
-                })
+            .map(|level| {
+                level
+                    .l0
+                    .sub_levels
+                    .iter()
+                    .filter(|sub_level| {
+                        sub_level.level_type == LevelType::Nonoverlapping
+                            && sub_level.vnode_partition_count > 0
+                    })
+                    .count()
             })
             .unwrap_or(0);
         metrics
@@ -481,16 +481,16 @@ pub fn trigger_lsm_stat(
     {
         // compact_level_compression_ratio
         let level_compression_ratio = levels
-            .get_levels()
+            .levels
             .iter()
             .map(|level| {
-                let ratio = if level.get_uncompressed_file_size() == 0 {
+                let ratio = if level.uncompressed_file_size == 0 {
                     0.0
                 } else {
-                    level.get_total_file_size() as f64 / level.get_uncompressed_file_size() as f64
+                    level.total_file_size as f64 / level.uncompressed_file_size as f64
                 };
 
-                (level.get_level_idx(), ratio)
+                (level.level_idx, ratio)
             })
             .collect_vec();
 
diff --git a/src/meta/src/hummock/mock_hummock_meta_client.rs b/src/meta/src/hummock/mock_hummock_meta_client.rs
index dea226e28b047..c821e4c911dc3 100644
--- a/src/meta/src/hummock/mock_hummock_meta_client.rs
+++ b/src/meta/src/hummock/mock_hummock_meta_client.rs
@@ -22,8 +22,11 @@ use async_trait::async_trait;
 use fail::fail_point;
 use futures::stream::BoxStream;
 use futures::{Stream, StreamExt};
+use itertools::Itertools;
 use risingwave_hummock_sdk::change_log::build_table_change_log_delta;
+use risingwave_hummock_sdk::compact_task::CompactTask;
 use risingwave_hummock_sdk::compaction_group::StaticCompactionGroupId;
+use risingwave_hummock_sdk::sstable_info::SstableInfo;
 use risingwave_hummock_sdk::version::HummockVersion;
 use risingwave_hummock_sdk::{
     HummockContextId, HummockEpoch, HummockSstableObjectId, HummockVersionId, LocalSstableInfo,
@@ -34,7 +37,7 @@ use risingwave_pb::hummock::compact_task::TaskStatus;
 use risingwave_pb::hummock::subscribe_compaction_event_request::{Event, ReportTask};
 use risingwave_pb::hummock::subscribe_compaction_event_response::Event as ResponseEvent;
 use risingwave_pb::hummock::{
-    compact_task, CompactTask, HummockSnapshot, SubscribeCompactionEventRequest,
+    compact_task, HummockSnapshot, PbHummockVersion, SubscribeCompactionEventRequest,
     SubscribeCompactionEventResponse, VacuumTask,
 };
 use risingwave_rpc_client::error::{Result, RpcError};
@@ -92,6 +95,10 @@ impl MockHummockMetaClient {
             .await
             .unwrap_or(None)
     }
+
+    pub fn context_id(&self) -> HummockContextId {
+        self.context_id
+    }
 }
 
 fn mock_err(error: super::error::Error) -> RpcError {
@@ -162,7 +169,7 @@ impl HummockMetaClient for MockHummockMetaClient {
         let sst_to_worker = sync_result
             .uncommitted_ssts
             .iter()
-            .map(|LocalSstableInfo { sst_info, .. }| (sst_info.get_object_id(), self.context_id))
+            .map(|LocalSstableInfo { sst_info, .. }| (sst_info.object_id, self.context_id))
             .collect();
         let new_table_watermark = sync_result.table_watermarks;
         let table_change_log = build_table_change_log_delta(
@@ -180,11 +187,7 @@ impl HummockMetaClient for MockHummockMetaClient {
         );
         self.hummock_manager
             .commit_epoch(CommitEpochInfo::new(
-                sync_result
-                    .uncommitted_ssts
-                    .into_iter()
-                    .map(|sst| sst.into())
-                    .collect(),
+                sync_result.uncommitted_ssts,
                 new_table_watermark,
                 sst_to_worker,
                 None,
@@ -194,6 +197,7 @@ impl HummockMetaClient for MockHummockMetaClient {
                     version.state_table_info.info().keys().cloned().collect(),
                 )]),
                 epoch,
+                vec![],
             ))
             .await
             .map_err(mock_err)?;
@@ -228,7 +232,11 @@ impl HummockMetaClient for MockHummockMetaClient {
         unimplemented!()
     }
 
-    async fn trigger_full_gc(&self, _sst_retention_time_sec: u64) -> Result<()> {
+    async fn trigger_full_gc(
+        &self,
+        _sst_retention_time_sec: u64,
+        _prefix: Option<String>,
+    ) -> Result<()> {
         unimplemented!()
     }
 
@@ -293,7 +301,7 @@ impl HummockMetaClient for MockHummockMetaClient {
                     .unwrap()
                 {
                     let resp = SubscribeCompactionEventResponse {
-                        event: Some(ResponseEvent::CompactTask(task)),
+                        event: Some(ResponseEvent::CompactTask(task.into())),
                         create_at: SystemTime::now()
                             .duration_since(std::time::UNIX_EPOCH)
                             .expect("Clock may have gone backwards")
@@ -323,7 +331,10 @@ impl HummockMetaClient for MockHummockMetaClient {
                             .report_compact_task(
                                 task_id,
                                 TaskStatus::try_from(task_status).unwrap(),
-                                sorted_output_ssts,
+                                sorted_output_ssts
+                                    .into_iter()
+                                    .map(SstableInfo::from)
+                                    .collect_vec(),
                                 Some(table_stats_change),
                             )
                             .await
@@ -345,6 +356,10 @@ impl HummockMetaClient for MockHummockMetaClient {
             }),
         ))
     }
+
+    async fn get_version_by_epoch(&self, _epoch: HummockEpoch) -> Result<PbHummockVersion> {
+        unimplemented!()
+    }
 }
 
 impl MockHummockMetaClient {
diff --git a/src/meta/src/hummock/model/ext/hummock.rs b/src/meta/src/hummock/model/ext/hummock.rs
index 5cd1eb08402ec..c82df1013737e 100644
--- a/src/meta/src/hummock/model/ext/hummock.rs
+++ b/src/meta/src/hummock/model/ext/hummock.rs
@@ -50,7 +50,7 @@ impl From<sea_orm::DbErr> for MetadataModelError {
 impl Transactional<Transaction> for CompactionGroup {
     async fn upsert_in_transaction(&self, trx: &mut Transaction) -> MetadataModelResult<()> {
         let m = compaction_config::ActiveModel {
-            compaction_group_id: Set(self.group_id as _),
+            compaction_group_id: Set(self.group_id.try_into().unwrap()),
             config: Set(CompactionConfig::from(&(*self.compaction_config))),
         };
         compaction_config::Entity::insert(m)
@@ -65,9 +65,11 @@ impl Transactional<Transaction> for CompactionGroup {
     }
 
     async fn delete_in_transaction(&self, trx: &mut Transaction) -> MetadataModelResult<()> {
-        compaction_config::Entity::delete_by_id(self.group_id as CompactionGroupId)
-            .exec(trx)
-            .await?;
+        compaction_config::Entity::delete_by_id(
+            CompactionGroupId::try_from(self.group_id).unwrap(),
+        )
+        .exec(trx)
+        .await?;
         Ok(())
     }
 }
@@ -76,7 +78,7 @@ impl Transactional<Transaction> for CompactionGroup {
 impl Transactional<Transaction> for CompactStatus {
     async fn upsert_in_transaction(&self, trx: &mut Transaction) -> MetadataModelResult<()> {
         let m = compaction_status::ActiveModel {
-            compaction_group_id: Set(self.compaction_group_id as _),
+            compaction_group_id: Set(self.compaction_group_id.try_into().unwrap()),
             status: Set(LevelHandlers::from(
                 self.level_handlers.iter().map_into().collect_vec(),
             )),
@@ -93,9 +95,11 @@ impl Transactional<Transaction> for CompactStatus {
     }
 
     async fn delete_in_transaction(&self, trx: &mut Transaction) -> MetadataModelResult<()> {
-        compaction_status::Entity::delete_by_id(self.compaction_group_id as CompactionGroupId)
-            .exec(trx)
-            .await?;
+        compaction_status::Entity::delete_by_id(
+            CompactionGroupId::try_from(self.compaction_group_id).unwrap(),
+        )
+        .exec(trx)
+        .await?;
         Ok(())
     }
 }
@@ -105,8 +109,8 @@ impl Transactional<Transaction> for CompactTaskAssignment {
     async fn upsert_in_transaction(&self, trx: &mut Transaction) -> MetadataModelResult<()> {
         let task = self.compact_task.to_owned().unwrap();
         let m = compaction_task::ActiveModel {
-            id: Set(task.task_id as _),
-            context_id: Set(self.context_id as _),
+            id: Set(task.task_id.try_into().unwrap()),
+            context_id: Set(self.context_id.try_into().unwrap()),
             task: Set(CompactionTask::from(&task)),
         };
         compaction_task::Entity::insert(m)
@@ -125,7 +129,7 @@ impl Transactional<Transaction> for CompactTaskAssignment {
 
     async fn delete_in_transaction(&self, trx: &mut Transaction) -> MetadataModelResult<()> {
         compaction_task::Entity::delete_by_id(
-            self.compact_task.as_ref().unwrap().task_id as CompactionTaskId,
+            CompactionTaskId::try_from(self.compact_task.as_ref().unwrap().task_id).unwrap(),
         )
         .exec(trx)
         .await?;
@@ -137,8 +141,8 @@ impl Transactional<Transaction> for CompactTaskAssignment {
 impl Transactional<Transaction> for HummockPinnedVersion {
     async fn upsert_in_transaction(&self, trx: &mut Transaction) -> MetadataModelResult<()> {
         let m = hummock_pinned_version::ActiveModel {
-            context_id: Set(self.context_id as _),
-            min_pinned_id: Set(self.min_pinned_id as _),
+            context_id: Set(self.context_id.try_into().unwrap()),
+            min_pinned_id: Set(self.min_pinned_id.try_into().unwrap()),
         };
         hummock_pinned_version::Entity::insert(m)
             .on_conflict(
@@ -152,7 +156,7 @@ impl Transactional<Transaction> for HummockPinnedVersion {
     }
 
     async fn delete_in_transaction(&self, trx: &mut Transaction) -> MetadataModelResult<()> {
-        hummock_pinned_version::Entity::delete_by_id(self.context_id as WorkerId)
+        hummock_pinned_version::Entity::delete_by_id(WorkerId::try_from(self.context_id).unwrap())
             .exec(trx)
             .await?;
         Ok(())
@@ -163,8 +167,8 @@ impl Transactional<Transaction> for HummockPinnedVersion {
 impl Transactional<Transaction> for HummockPinnedSnapshot {
     async fn upsert_in_transaction(&self, trx: &mut Transaction) -> MetadataModelResult<()> {
         let m = hummock_pinned_snapshot::ActiveModel {
-            context_id: Set(self.context_id as _),
-            min_pinned_snapshot: Set(self.minimal_pinned_snapshot as _),
+            context_id: Set(self.context_id.try_into().unwrap()),
+            min_pinned_snapshot: Set(self.minimal_pinned_snapshot.try_into().unwrap()),
         };
         hummock_pinned_snapshot::Entity::insert(m)
             .on_conflict(
@@ -178,7 +182,7 @@ impl Transactional<Transaction> for HummockPinnedSnapshot {
     }
 
     async fn delete_in_transaction(&self, trx: &mut Transaction) -> MetadataModelResult<()> {
-        hummock_pinned_snapshot::Entity::delete_by_id(self.context_id as i32)
+        hummock_pinned_snapshot::Entity::delete_by_id(WorkerId::try_from(self.context_id).unwrap())
             .exec(trx)
             .await?;
         Ok(())
@@ -189,7 +193,7 @@ impl Transactional<Transaction> for HummockPinnedSnapshot {
 impl Transactional<Transaction> for HummockVersionStats {
     async fn upsert_in_transaction(&self, trx: &mut Transaction) -> MetadataModelResult<()> {
         let m = hummock_version_stats::ActiveModel {
-            id: Set(self.hummock_version_id as _),
+            id: Set(self.hummock_version_id.try_into().unwrap()),
             stats: Set(TableStats(self.table_stats.clone())),
         };
         hummock_version_stats::Entity::insert(m)
@@ -204,9 +208,11 @@ impl Transactional<Transaction> for HummockVersionStats {
     }
 
     async fn delete_in_transaction(&self, trx: &mut Transaction) -> MetadataModelResult<()> {
-        hummock_version_stats::Entity::delete_by_id(self.hummock_version_id as i64)
-            .exec(trx)
-            .await?;
+        hummock_version_stats::Entity::delete_by_id(
+            HummockVersionId::try_from(self.hummock_version_id).unwrap(),
+        )
+        .exec(trx)
+        .await?;
         Ok(())
     }
 }
@@ -215,12 +221,12 @@ impl Transactional<Transaction> for HummockVersionStats {
 impl Transactional<Transaction> for HummockVersionDelta {
     async fn upsert_in_transaction(&self, trx: &mut Transaction) -> MetadataModelResult<()> {
         let m = hummock_version_delta::ActiveModel {
-            id: Set(self.id as _),
-            prev_id: Set(self.prev_id as _),
-            max_committed_epoch: Set(self.max_committed_epoch as _),
-            safe_epoch: Set(self.safe_epoch as _),
+            id: Set(self.id.try_into().unwrap()),
+            prev_id: Set(self.prev_id.try_into().unwrap()),
+            max_committed_epoch: Set(self.max_committed_epoch.try_into().unwrap()),
+            safe_epoch: Set(self.visible_table_safe_epoch().try_into().unwrap()),
             trivial_move: Set(self.trivial_move),
-            full_version_delta: Set(FullVersionDelta::from(&self.to_protobuf())),
+            full_version_delta: Set(FullVersionDelta::from(&self.into())),
         };
         hummock_version_delta::Entity::insert(m)
             .on_conflict(
@@ -240,7 +246,7 @@ impl Transactional<Transaction> for HummockVersionDelta {
     }
 
     async fn delete_in_transaction(&self, trx: &mut Transaction) -> MetadataModelResult<()> {
-        hummock_version_delta::Entity::delete_by_id(self.id as HummockVersionId)
+        hummock_version_delta::Entity::delete_by_id(HummockVersionId::try_from(self.id).unwrap())
             .exec(trx)
             .await?;
         Ok(())
@@ -249,14 +255,17 @@ impl Transactional<Transaction> for HummockVersionDelta {
 
 impl From<compaction_config::Model> for CompactionGroup {
     fn from(value: compaction_config::Model) -> Self {
-        Self::new(value.compaction_group_id as _, value.config.to_protobuf())
+        Self::new(
+            value.compaction_group_id.try_into().unwrap(),
+            value.config.to_protobuf(),
+        )
     }
 }
 
 impl From<compaction_status::Model> for CompactStatus {
     fn from(value: compaction_status::Model) -> Self {
         Self {
-            compaction_group_id: value.compaction_group_id as _,
+            compaction_group_id: value.compaction_group_id.try_into().unwrap(),
             level_handlers: value.status.to_protobuf().iter().map_into().collect(),
         }
     }
diff --git a/src/meta/src/hummock/model/version_delta.rs b/src/meta/src/hummock/model/version_delta.rs
index 1a87b9d456989..ed2be4761acaa 100644
--- a/src/meta/src/hummock/model/version_delta.rs
+++ b/src/meta/src/hummock/model/version_delta.rs
@@ -29,7 +29,7 @@ impl MetadataModel for HummockVersionDelta {
     }
 
     fn to_protobuf(&self) -> Self::PbType {
-        self.to_protobuf()
+        self.into()
     }
 
     fn from_protobuf(prost: Self::PbType) -> Self {
diff --git a/src/meta/src/hummock/test_utils.rs b/src/meta/src/hummock/test_utils.rs
index 32cbd4932adf6..9a764f871cd63 100644
--- a/src/meta/src/hummock/test_utils.rs
+++ b/src/meta/src/hummock/test_utils.rs
@@ -14,25 +14,35 @@
 
 #![cfg(any(test, feature = "test"))]
 
+use std::collections::{BTreeSet, HashMap};
 use std::sync::Arc;
 use std::time::Duration;
 
+use bytes::Bytes;
 use itertools::Itertools;
-use risingwave_common::catalog::TableId;
+use risingwave_common::catalog::{TableId, TableOption};
 use risingwave_common::util::epoch::test_epoch;
 use risingwave_hummock_sdk::compaction_group::StaticCompactionGroupId;
 use risingwave_hummock_sdk::key::key_with_epoch;
-use risingwave_hummock_sdk::version::HummockVersion;
+use risingwave_hummock_sdk::key_range::KeyRange;
+use risingwave_hummock_sdk::level::Levels;
+use risingwave_hummock_sdk::sstable_info::SstableInfo;
+use risingwave_hummock_sdk::table_watermark::TableWatermarks;
+use risingwave_hummock_sdk::version::{HummockVersion, HummockVersionStateTableInfo};
 use risingwave_hummock_sdk::{
     CompactionGroupId, HummockContextId, HummockEpoch, HummockSstableObjectId, LocalSstableInfo,
 };
 use risingwave_pb::common::{HostAddress, WorkerNode, WorkerType};
 use risingwave_pb::hummock::compact_task::TaskStatus;
-use risingwave_pb::hummock::{CompactionConfig, KeyRange, SstableInfo};
+use risingwave_pb::hummock::CompactionConfig;
 use risingwave_pb::meta::add_worker_node_request::Property;
 
 use crate::hummock::compaction::compaction_config::CompactionConfigBuilder;
-use crate::hummock::compaction::selector::default_compaction_selector;
+use crate::hummock::compaction::selector::{default_compaction_selector, LocalSelectorStatistic};
+use crate::hummock::compaction::{CompactionDeveloperConfig, CompactionSelectorContext};
+use crate::hummock::level_handler::LevelHandler;
+pub use crate::hummock::manager::CommitEpochInfo;
+use crate::hummock::model::CompactionGroup;
 use crate::hummock::{CompactorManager, HummockManager, HummockManagerRef};
 use crate::manager::{
     ClusterManager, ClusterManagerRef, FragmentManager, MetaSrvEnv, META_NODE_ID,
@@ -41,12 +51,7 @@ use crate::rpc::metrics::MetaMetrics;
 
 pub fn to_local_sstable_info(ssts: &[SstableInfo]) -> Vec<LocalSstableInfo> {
     ssts.iter()
-        .map(|sst| {
-            LocalSstableInfo::with_compaction_group(
-                StaticCompactionGroupId::StateDefault.into(),
-                sst.clone(),
-            )
-        })
+        .map(|sst| LocalSstableInfo::for_test(sst.clone()))
         .collect_vec()
 }
 
@@ -70,7 +75,7 @@ pub async fn add_test_tables(
     let ssts = to_local_sstable_info(&test_tables);
     let sst_to_worker = ssts
         .iter()
-        .map(|LocalSstableInfo { sst_info, .. }| (sst_info.get_object_id(), context_id))
+        .map(|LocalSstableInfo { sst_info, .. }| (sst_info.object_id, context_id))
         .collect();
     hummock_manager
         .commit_epoch_for_test(epoch, ssts, sst_to_worker)
@@ -145,7 +150,7 @@ pub async fn add_test_tables(
     let ssts = to_local_sstable_info(&test_tables_3);
     let sst_to_worker = ssts
         .iter()
-        .map(|LocalSstableInfo { sst_info, .. }| (sst_info.get_object_id(), context_id))
+        .map(|LocalSstableInfo { sst_info, .. }| (sst_info.object_id, context_id))
         .collect();
     hummock_manager
         .commit_epoch_for_test(epoch, ssts, sst_to_worker)
@@ -164,21 +169,21 @@ pub fn generate_test_sstables_with_table_id(
         sst_info.push(SstableInfo {
             object_id: sst_id,
             sst_id,
-            key_range: Some(KeyRange {
-                left: key_with_epoch(
+            key_range: KeyRange {
+                left: Bytes::from(key_with_epoch(
                     format!("{:03}\0\0_key_test_{:05}", table_id, i + 1)
                         .as_bytes()
                         .to_vec(),
                     epoch,
-                ),
-                right: key_with_epoch(
+                )),
+                right: Bytes::from(key_with_epoch(
                     format!("{:03}\0\0_key_test_{:05}", table_id, (i + 1) * 10)
                         .as_bytes()
                         .to_vec(),
                     epoch,
-                ),
+                )),
                 right_exclusive: false,
-            }),
+            },
             file_size: 2,
             table_ids: vec![table_id],
             uncompressed_file_size: 2,
@@ -195,11 +200,11 @@ pub fn generate_test_tables(epoch: u64, sst_ids: Vec<HummockSstableObjectId>) ->
         sst_info.push(SstableInfo {
             object_id: sst_id,
             sst_id,
-            key_range: Some(KeyRange {
-                left: iterator_test_key_of_epoch(sst_id, i + 1, epoch),
-                right: iterator_test_key_of_epoch(sst_id, (i + 1) * 10, epoch),
+            key_range: KeyRange {
+                left: Bytes::from(iterator_test_key_of_epoch(sst_id, i + 1, epoch)),
+                right: Bytes::from(iterator_test_key_of_epoch(sst_id, (i + 1) * 10, epoch)),
                 right_exclusive: false,
-            }),
+            },
             file_size: 2,
             table_ids: vec![sst_id as u32, sst_id as u32 * 10000],
             uncompressed_file_size: 2,
@@ -274,7 +279,7 @@ pub fn iterator_test_key_of_epoch(
 pub fn get_sorted_object_ids(sstables: &[SstableInfo]) -> Vec<HummockSstableObjectId> {
     sstables
         .iter()
-        .map(|table| table.get_object_id())
+        .map(|table| table.object_id)
         .sorted()
         .collect_vec()
 }
@@ -292,8 +297,8 @@ pub fn get_sorted_committed_object_ids(
     levels
         .levels
         .iter()
-        .chain(levels.l0.as_ref().unwrap().sub_levels.iter())
-        .flat_map(|levels| levels.table_infos.iter().map(|info| info.get_object_id()))
+        .chain(levels.l0.sub_levels.iter())
+        .flat_map(|levels| levels.table_infos.iter().map(|info| info.object_id))
         .sorted()
         .collect_vec()
 }
@@ -382,7 +387,7 @@ pub async fn commit_from_meta_node(
 ) -> crate::hummock::error::Result<()> {
     let sst_to_worker = ssts
         .iter()
-        .map(|LocalSstableInfo { sst_info, .. }| (sst_info.get_object_id(), META_NODE_ID))
+        .map(|LocalSstableInfo { sst_info, .. }| (sst_info.object_id, META_NODE_ID))
         .collect();
     hummock_manager_ref
         .commit_epoch_for_test(epoch, ssts, sst_to_worker)
@@ -399,7 +404,7 @@ pub async fn add_ssts(
     let ssts = to_local_sstable_info(&test_tables);
     let sst_to_worker = ssts
         .iter()
-        .map(|LocalSstableInfo { sst_info, .. }| (sst_info.get_object_id(), context_id))
+        .map(|LocalSstableInfo { sst_info, .. }| (sst_info.object_id, context_id))
         .collect();
     hummock_manager
         .commit_epoch_for_test(epoch, ssts, sst_to_worker)
@@ -407,3 +412,27 @@ pub async fn add_ssts(
         .unwrap();
     test_tables
 }
+
+pub fn compaction_selector_context<'a>(
+    group: &'a CompactionGroup,
+    levels: &'a Levels,
+    member_table_ids: &'a BTreeSet<TableId>,
+    level_handlers: &'a mut [LevelHandler],
+    selector_stats: &'a mut LocalSelectorStatistic,
+    table_id_to_options: &'a HashMap<u32, TableOption>,
+    developer_config: Arc<CompactionDeveloperConfig>,
+    table_watermarks: &'a HashMap<TableId, Arc<TableWatermarks>>,
+    state_table_info: &'a HummockVersionStateTableInfo,
+) -> CompactionSelectorContext<'a> {
+    CompactionSelectorContext {
+        group,
+        levels,
+        member_table_ids,
+        level_handlers,
+        selector_stats,
+        table_id_to_options,
+        developer_config,
+        table_watermarks,
+        state_table_info,
+    }
+}
diff --git a/src/meta/src/hummock/vacuum.rs b/src/meta/src/hummock/vacuum.rs
index 6cde13507836b..d747651b86d43 100644
--- a/src/meta/src/hummock/vacuum.rs
+++ b/src/meta/src/hummock/vacuum.rs
@@ -17,6 +17,8 @@ use std::sync::Arc;
 use std::time::Duration;
 
 use itertools::Itertools;
+use risingwave_common::system_param::reader::SystemParamsRead;
+use risingwave_common::util::epoch::Epoch;
 use risingwave_hummock_sdk::HummockSstableObjectId;
 use risingwave_pb::hummock::subscribe_compaction_event_response::Event as ResponseEvent;
 use risingwave_pb::hummock::VacuumTask;
@@ -76,6 +78,21 @@ impl VacuumManager {
                 break;
             }
         }
+
+        let current_epoch_time = Epoch::now().physical_time();
+        let epoch_watermark = Epoch::from_physical_time(
+            current_epoch_time.saturating_sub(
+                self.env
+                    .system_params_reader()
+                    .await
+                    .time_travel_retention_ms(),
+            ),
+        )
+        .0;
+        self.hummock_manager
+            .truncate_time_travel_metadata(epoch_watermark)
+            .await?;
+
         Ok(total_deleted)
     }
 
@@ -165,6 +182,9 @@ impl VacuumManager {
         &self,
         objects_to_delete: &mut Vec<HummockSstableObjectId>,
     ) -> MetaResult<()> {
+        if objects_to_delete.is_empty() {
+            return Ok(());
+        }
         let reject = self.backup_manager.list_pinned_ssts();
         // Ack these SSTs immediately, because they tend to be pinned for long time.
         // They will be GCed during full GC when they are no longer pinned.
@@ -258,7 +278,7 @@ mod tests {
                     .first()
                     .unwrap()
                     .iter()
-                    .map(|s| s.get_object_id())
+                    .map(|s| s.object_id)
                     .collect_vec(),
             })
             .await
diff --git a/src/meta/src/lib.rs b/src/meta/src/lib.rs
index 71c99a7e065b4..811b3b152d061 100644
--- a/src/meta/src/lib.rs
+++ b/src/meta/src/lib.rs
@@ -20,7 +20,6 @@
 #![feature(extract_if)]
 #![feature(hash_extract_if)]
 #![feature(btree_extract_if)]
-#![feature(lazy_cell)]
 #![feature(let_chains)]
 #![feature(error_generic_member_access)]
 #![feature(assert_matches)]
diff --git a/src/meta/src/manager/catalog/database.rs b/src/meta/src/manager/catalog/database.rs
index 299065fe9586d..da1238d43d54b 100644
--- a/src/meta/src/manager/catalog/database.rs
+++ b/src/meta/src/manager/catalog/database.rs
@@ -26,13 +26,14 @@ use risingwave_pb::catalog::{
 };
 use risingwave_pb::data::DataType;
 use risingwave_pb::user::grant_privilege::PbObject;
+use tokio::sync::oneshot::Sender;
 
 use super::utils::{get_refed_secret_ids_from_sink, get_refed_secret_ids_from_source};
 use super::{
     ConnectionId, DatabaseId, FunctionId, RelationId, SchemaId, SecretId, SinkId, SourceId,
     SubscriptionId, ViewId,
 };
-use crate::manager::{IndexId, MetaSrvEnv, TableId, UserId};
+use crate::manager::{IndexId, MetaSrvEnv, NotificationVersion, TableId, UserId};
 use crate::model::MetadataModel;
 use crate::{MetaError, MetaResult};
 
@@ -95,6 +96,13 @@ pub struct DatabaseManager {
     pub(super) in_progress_creation_streaming_job: HashMap<TableId, RelationKey>,
     // In-progress creating tables, including internal tables.
     pub(super) in_progress_creating_tables: HashMap<TableId, Table>,
+
+    /// Registered finish notifiers for creating tables.
+    ///
+    /// `DdlController` will update this map, and pass the `tx` side to `CatalogController`.
+    /// On notifying, we can remove the entry from this map.
+    pub creating_table_finish_notifier:
+        HashMap<TableId, Vec<Sender<MetaResult<NotificationVersion>>>>,
 }
 
 impl DatabaseManager {
@@ -187,6 +195,7 @@ impl DatabaseManager {
             in_progress_creation_tracker: HashSet::default(),
             in_progress_creation_streaming_job: HashMap::default(),
             in_progress_creating_tables: HashMap::default(),
+            creating_table_finish_notifier: Default::default(),
         })
     }
 
@@ -246,7 +255,7 @@ impl DatabaseManager {
                 && x.name.eq(&relation_key.2)
         }) {
             if t.stream_job_status == StreamJobStatus::Creating as i32 {
-                bail!("table is in creating procedure, table id: {}", t.id);
+                bail!("The table is being created, table id: {}", t.id);
             } else {
                 Err(MetaError::catalog_duplicated("table", &relation_key.2))
             }
@@ -567,6 +576,16 @@ impl DatabaseManager {
 
     pub fn unmark_creating_streaming_job(&mut self, table_id: TableId) {
         self.in_progress_creation_streaming_job.remove(&table_id);
+        for tx in self
+            .creating_table_finish_notifier
+            .remove(&table_id)
+            .into_iter()
+            .flatten()
+        {
+            let _ = tx.send(Err(MetaError::cancelled(format!(
+                "streaing_job {table_id} has been cancelled"
+            ))));
+        }
     }
 
     pub fn find_creating_streaming_job_id(&self, key: &RelationKey) -> Option<TableId> {
diff --git a/src/meta/src/manager/catalog/fragment.rs b/src/meta/src/manager/catalog/fragment.rs
index 537e2fac01f22..a523bbfeb3e7e 100644
--- a/src/meta/src/manager/catalog/fragment.rs
+++ b/src/meta/src/manager/catalog/fragment.rs
@@ -12,24 +12,25 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::{BTreeMap, HashMap, HashSet};
+use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
 use std::sync::Arc;
 
 use anyhow::{anyhow, Context};
 use itertools::Itertools;
 use risingwave_common::bail;
-use risingwave_common::buffer::Bitmap;
+use risingwave_common::bitmap::Bitmap;
 use risingwave_common::catalog::TableId;
-use risingwave_common::hash::{ActorMapping, ParallelUnitId, ParallelUnitMapping};
+use risingwave_common::hash::{ActorMapping, WorkerSlotId, WorkerSlotMapping};
 use risingwave_common::util::stream_graph_visitor::{
     visit_stream_node, visit_stream_node_cont, visit_stream_node_cont_mut,
 };
 use risingwave_common::util::worker_util::WorkerNodeId;
 use risingwave_connector::source::SplitImpl;
 use risingwave_meta_model_v2::SourceId;
-use risingwave_pb::common::{PbParallelUnitMapping, PbWorkerSlotMapping};
+use risingwave_pb::common::PbActorLocation;
 use risingwave_pb::meta::subscribe_response::{Info, Operation};
 use risingwave_pb::meta::table_fragments::actor_status::ActorState;
+use risingwave_pb::meta::table_fragments::fragment::FragmentDistributionType;
 use risingwave_pb::meta::table_fragments::{ActorStatus, Fragment, State};
 use risingwave_pb::meta::FragmentWorkerSlotMapping;
 use risingwave_pb::stream_plan::stream_node::NodeBody;
@@ -61,24 +62,76 @@ impl FragmentManagerCore {
     pub fn all_running_fragment_mappings(
         &self,
     ) -> impl Iterator<Item = FragmentWorkerSlotMapping> + '_ {
-        self.table_fragments
+        let mut result = vec![];
+
+        for table_fragment in self
+            .table_fragments
             .values()
             .filter(|tf| tf.state() != State::Initial)
-            .flat_map(|table_fragments| {
-                table_fragments
-                    .fragments
-                    .values()
-                    .map(move |fragment| FragmentWorkerSlotMapping {
+        {
+            result.extend(Self::extract_fragment_mapping(table_fragment).into_iter());
+        }
+
+        result.into_iter()
+    }
+
+    fn extract_fragment_mapping(table_fragment: &TableFragments) -> Vec<FragmentWorkerSlotMapping> {
+        let mut result = Vec::with_capacity(table_fragment.fragments.len());
+        for fragment in table_fragment.fragments.values() {
+            match fragment.get_distribution_type().unwrap() {
+                FragmentDistributionType::Unspecified => unreachable!(),
+                FragmentDistributionType::Single => {
+                    let actor = fragment
+                        .get_actors()
+                        .iter()
+                        .exactly_one()
+                        .expect("single actor");
+                    let status = table_fragment.actor_status.get(&actor.actor_id).unwrap();
+                    let worker_id = status.worker_id();
+                    result.push(FragmentWorkerSlotMapping {
                         fragment_id: fragment.fragment_id,
                         mapping: Some(
-                            FragmentManager::convert_mapping(
-                                &table_fragments.actor_status,
-                                fragment.vnode_mapping.as_ref().unwrap(),
-                            )
-                            .unwrap(),
+                            WorkerSlotMapping::new_single(WorkerSlotId::new(worker_id, 0))
+                                .to_protobuf(),
                         ),
-                    })
-            })
+                    });
+                }
+                FragmentDistributionType::Hash => {
+                    let mut actor_bitmaps = HashMap::new();
+                    let mut actor_to_workers = HashMap::new();
+
+                    for actor in &fragment.actors {
+                        let status = table_fragment.actor_status.get(&actor.actor_id).unwrap();
+
+                        match status.state() {
+                            ActorState::Unspecified => unreachable!(),
+                            // skip inactive actors
+                            ActorState::Inactive => continue,
+                            ActorState::Running => {}
+                        }
+
+                        let worker_id = status.worker_id();
+                        actor_bitmaps.insert(
+                            actor.actor_id as ActorId,
+                            Bitmap::from(actor.vnode_bitmap.as_ref().unwrap()),
+                        );
+
+                        actor_to_workers.insert(actor.actor_id, worker_id);
+                    }
+
+                    let actor_mapping = ActorMapping::from_bitmaps(&actor_bitmaps);
+
+                    let mapping = actor_mapping.to_worker_slot(&actor_to_workers);
+
+                    result.push(FragmentWorkerSlotMapping {
+                        fragment_id: fragment.fragment_id,
+                        mapping: Some(mapping.to_protobuf()),
+                    });
+                }
+            }
+        }
+
+        result
     }
 
     fn running_fragment_parallelisms(
@@ -95,16 +148,13 @@ impl FragmentManagerCore {
                     {
                         return None;
                     }
-                    let parallelism = match fragment.vnode_mapping.as_ref() {
-                        None => {
-                            tracing::warn!(
-                                "vnode mapping for fragment {} not found",
-                                fragment.fragment_id
-                            );
-                            1
-                        }
-                        Some(m) => ParallelUnitMapping::from_protobuf(m).iter_unique().count(),
+
+                    let parallelism = match fragment.get_distribution_type().unwrap() {
+                        FragmentDistributionType::Unspecified => unreachable!(),
+                        FragmentDistributionType::Single => 1,
+                        FragmentDistributionType::Hash => fragment.get_actors().len(),
                     };
+
                     Some((fragment.fragment_id, parallelism))
                 })
             })
@@ -123,7 +173,7 @@ pub struct FragmentManager {
     core: RwLock<FragmentManagerCore>,
 }
 
-#[derive(Clone)]
+#[derive(Clone, Debug)]
 pub struct InflightFragmentInfo {
     pub actors: HashMap<ActorId, WorkerNodeId>,
     pub state_table_ids: HashSet<TableId>,
@@ -209,33 +259,10 @@ impl FragmentManager {
     }
 
     async fn notify_fragment_mapping(&self, table_fragment: &TableFragments, operation: Operation) {
-        // Notify all fragment mapping to frontend nodes
-        for fragment in table_fragment.fragments.values() {
-            let vnode_mapping = fragment
-                .vnode_mapping
-                .as_ref()
-                .expect("no data distribution found");
-
-            let fragment_mapping = if let Operation::Delete = operation {
-                FragmentWorkerSlotMapping {
-                    fragment_id: fragment.fragment_id,
-                    mapping: None,
-                }
-            } else {
-                FragmentWorkerSlotMapping {
-                    fragment_id: fragment.fragment_id,
-                    mapping: Some(
-                        Self::convert_mapping(&table_fragment.actor_status, vnode_mapping).unwrap(),
-                    ),
-                }
-            };
-
+        for mapping in FragmentManagerCore::extract_fragment_mapping(table_fragment) {
             self.env
                 .notification_manager()
-                .notify_frontend(
-                    operation,
-                    Info::StreamingWorkerSlotMapping(fragment_mapping),
-                )
+                .notify_frontend(operation, Info::StreamingWorkerSlotMapping(mapping))
                 .await;
         }
 
@@ -320,7 +347,7 @@ impl FragmentManager {
         let map = &mut guard.table_fragments;
         let table_id = table_fragment.table_id();
         if map.contains_key(&table_id) {
-            bail!("table_fragment already exist: id={}", table_id);
+            bail!("table_fragment already exists: id={}", table_id);
         }
 
         let mut table_fragments = BTreeMapTransaction::new(map);
@@ -677,6 +704,35 @@ impl FragmentManager {
 
         let mut dirty_downstream_table_ids = HashMap::new();
 
+        fn union_input_is_clean(
+            all_fragment_ids: &HashSet<FragmentId>,
+            input: &StreamNode,
+        ) -> bool {
+            match &input.node_body {
+                // for old version sink into table
+                Some(NodeBody::Merge(merge_node))
+                    if !all_fragment_ids.contains(&merge_node.upstream_fragment_id) =>
+                {
+                    false
+                }
+                // for new version sink into table with project
+                Some(NodeBody::Project(_)) => {
+                    let merge_stream_node = input.input.iter().exactly_one().expect(
+                        "project of the sink input for the target table should have only one input",
+                    );
+
+                    if let Some(NodeBody::Merge(merge_node)) = &merge_stream_node.node_body
+                        && !all_fragment_ids.contains(&merge_node.upstream_fragment_id)
+                    {
+                        false
+                    } else {
+                        true
+                    }
+                }
+                _ => true,
+            }
+        }
+
         for (table_id, table_fragment) in table_fragments.tree_ref() {
             if to_delete_table_ids.contains(table_id) {
                 continue;
@@ -695,9 +751,7 @@ impl FragmentManager {
                     visit_stream_node_cont(actor.nodes.as_ref().unwrap(), |node| {
                         if let Some(NodeBody::Union(_)) = node.node_body {
                             for input in &node.input {
-                                if let Some(NodeBody::Merge(merge_node)) = &input.node_body
-                                    && !all_fragment_ids.contains(&merge_node.upstream_fragment_id)
-                                {
+                                if !union_input_is_clean(&all_fragment_ids, input) {
                                     dirty_downstream_table_ids
                                         .insert(*table_id, fragment.fragment_id);
                                     return false;
@@ -727,15 +781,8 @@ impl FragmentManager {
             for actor in &mut fragment.actors {
                 visit_stream_node_cont_mut(actor.nodes.as_mut().unwrap(), |node| {
                     if let Some(NodeBody::Union(_)) = node.node_body {
-                        node.input.retain_mut(|input| {
-                            if let Some(NodeBody::Merge(merge_node)) = &mut input.node_body
-                                && !all_fragment_ids.contains(&merge_node.upstream_fragment_id)
-                            {
-                                false
-                            } else {
-                                true
-                            }
-                        });
+                        node.input
+                            .retain_mut(|input| union_input_is_clean(&all_fragment_ids, input));
                     }
                     true
                 })
@@ -762,13 +809,7 @@ impl FragmentManager {
                                 .get(&actor.actor_id)
                                 .expect("should exist");
                             if status.state == ActorState::Running as i32 {
-                                Some((
-                                    actor.actor_id,
-                                    status
-                                        .get_parallel_unit()
-                                        .expect("should set")
-                                        .worker_node_id,
-                                ))
+                                Some((actor.actor_id, status.worker_id()))
                             } else {
                                 None
                             }
@@ -801,14 +842,43 @@ impl FragmentManager {
             .get_mut(table_id)
             .with_context(|| format!("table_fragment not exist: id={}", table_id))?;
 
-        for status in table_fragment.actor_status.values_mut() {
-            if let Some(pu) = &status.parallel_unit
-                && migration_plan.parallel_unit_plan.contains_key(&pu.id)
-            {
-                status.parallel_unit = Some(migration_plan.parallel_unit_plan[&pu.id].clone());
+        let mut worker_fragment_map = HashMap::new();
+        for fragment in table_fragment.fragments() {
+            for actor in fragment.get_actors() {
+                let worker = table_fragment
+                    .actor_status
+                    .get(&actor.actor_id)
+                    .unwrap()
+                    .worker_id();
+
+                let fragment_ref = worker_fragment_map.entry(worker).or_insert(HashMap::new());
+                fragment_ref
+                    .entry(fragment.fragment_id)
+                    .or_insert(BTreeSet::new())
+                    .insert(actor.actor_id);
+            }
+        }
+
+        let mut rebuilt_actor_to_worker_slots = HashMap::new();
+
+        for (worker, fragment_map) in worker_fragment_map {
+            for (_, actors) in fragment_map {
+                for (idx, actor) in actors.into_iter().enumerate() {
+                    rebuilt_actor_to_worker_slots.insert(actor, WorkerSlotId::new(worker, idx));
+                }
+            }
+        }
+
+        for (actor, location) in rebuilt_actor_to_worker_slots {
+            if let Some(target) = migration_plan.worker_slot_plan.get(&location) {
+                let status = table_fragment
+                    .actor_status
+                    .get_mut(&actor)
+                    .expect("should exist");
+                status.location = PbActorLocation::from_worker(target.worker_id());
             }
         }
-        table_fragment.update_vnode_mapping(&migration_plan.parallel_unit_plan);
+
         let table_fragment = table_fragment.clone();
         let next_revision = current_revision.next();
 
@@ -826,6 +896,12 @@ impl FragmentManager {
     /// Used in [`crate::barrier::GlobalBarrierManager`]
     /// migrate actors and update fragments one by one according to the migration plan.
     pub async fn migrate_fragment_actors(&self, migration_plan: &MigrationPlan) -> MetaResult<()> {
+        let expired_workers: HashSet<_> = migration_plan
+            .worker_slot_plan
+            .keys()
+            .map(|w| w.worker_id())
+            .collect();
+
         let to_migrate_table_fragments = self
             .core
             .read()
@@ -834,9 +910,7 @@ impl FragmentManager {
             .values()
             .filter(|tf| {
                 for status in tf.actor_status.values() {
-                    if let Some(pu) = &status.parallel_unit
-                        && migration_plan.parallel_unit_plan.contains_key(&pu.id)
-                    {
+                    if expired_workers.contains(&status.worker_id()) {
                         return true;
                     }
                 }
@@ -853,21 +927,39 @@ impl FragmentManager {
         Ok(())
     }
 
-    pub async fn all_worker_parallel_units(&self) -> HashMap<WorkerId, HashSet<ParallelUnitId>> {
-        let mut all_worker_parallel_units = HashMap::new();
+    pub async fn all_worker_slots(&self) -> HashMap<WorkerId, HashSet<WorkerSlotId>> {
+        let mut all_worker_slots = HashMap::new();
         let map = &self.core.read().await.table_fragments;
+
+        let mut worker_fragment_map = HashMap::new();
+
         for table_fragment in map.values() {
-            table_fragment.worker_parallel_units().into_iter().for_each(
-                |(worker_id, parallel_units)| {
-                    all_worker_parallel_units
-                        .entry(worker_id)
-                        .or_insert_with(HashSet::new)
-                        .extend(parallel_units);
-                },
+            for fragment in table_fragment.fragments() {
+                for actor in fragment.get_actors() {
+                    let worker = table_fragment
+                        .actor_status
+                        .get(&actor.actor_id)
+                        .unwrap()
+                        .worker_id();
+
+                    let fragment_ref = worker_fragment_map.entry(worker).or_insert(HashMap::new());
+                    *fragment_ref.entry(fragment.fragment_id).or_insert(0) += 1;
+                }
+            }
+        }
+
+        for (worker, fragment_map) in worker_fragment_map {
+            let max_fragment = fragment_map.values().copied().max().unwrap();
+
+            all_worker_slots.insert(
+                worker,
+                (0..max_fragment)
+                    .map(|idx| WorkerSlotId::new(worker, idx as _))
+                    .collect(),
             );
         }
 
-        all_worker_parallel_units
+        all_worker_slots
     }
 
     pub async fn all_node_actors(
@@ -899,10 +991,8 @@ impl FragmentManager {
         let map = &self.core.read().await.table_fragments;
         for fragments in map.values() {
             for actor_status in fragments.actor_status.values() {
-                if let Some(pu) = &actor_status.parallel_unit {
-                    let e = actor_count.entry(pu.worker_node_id).or_insert(0);
-                    *e += 1;
-                }
+                let e = actor_count.entry(actor_status.worker_id()).or_insert(0);
+                *e += 1;
             }
         }
 
@@ -1270,12 +1360,12 @@ impl FragmentManager {
                 }
 
                 // update fragment's vnode mapping
-                let mut actor_to_parallel_unit = HashMap::with_capacity(fragment.actors.len());
+                let mut actor_to_worker = HashMap::with_capacity(fragment.actors.len());
                 let mut actor_to_vnode_bitmap = HashMap::with_capacity(fragment.actors.len());
                 for actor in &fragment.actors {
                     let actor_status = &actor_status[&actor.actor_id];
-                    let parallel_unit_id = actor_status.parallel_unit.as_ref().unwrap().id;
-                    actor_to_parallel_unit.insert(actor.actor_id, parallel_unit_id);
+                    let worker_id = actor_status.worker_id();
+                    actor_to_worker.insert(actor.actor_id, worker_id);
 
                     if let Some(vnode_bitmap) = &actor.vnode_bitmap {
                         let bitmap = Bitmap::from(vnode_bitmap);
@@ -1283,25 +1373,22 @@ impl FragmentManager {
                     }
                 }
 
-                let vnode_mapping = if actor_to_vnode_bitmap.is_empty() {
+                let worker_slot_mapping = if actor_to_vnode_bitmap.is_empty() {
                     // If there's no `vnode_bitmap`, then the fragment must be a singleton fragment.
-                    // We directly use the single parallel unit to construct the mapping.
+                    // We directly use the single worker slot to construct the mapping.
                     // TODO: also fill `vnode_bitmap` for the actor of singleton fragment so that we
                     // don't need this branch.
-                    let parallel_unit = *actor_to_parallel_unit.values().exactly_one().unwrap();
-                    ParallelUnitMapping::new_single(parallel_unit)
+
+                    let worker_id = *actor_to_worker.values().exactly_one().unwrap();
+                    WorkerSlotMapping::new_single(WorkerSlotId::new(worker_id, 0))
                 } else {
-                    // Generate the parallel unit mapping from the fragment's actor bitmaps.
-                    assert_eq!(actor_to_vnode_bitmap.len(), actor_to_parallel_unit.len());
+                    // Generate the worker slot mapping from the fragment's actor bitmaps with actor locations.
+                    assert_eq!(actor_to_vnode_bitmap.len(), actor_to_worker.len());
                     ActorMapping::from_bitmaps(&actor_to_vnode_bitmap)
-                        .to_parallel_unit(&actor_to_parallel_unit)
+                        .to_worker_slot(&actor_to_worker)
                 }
                 .to_protobuf();
 
-                *fragment.vnode_mapping.as_mut().unwrap() = vnode_mapping.clone();
-
-                let worker_slot_mapping = Self::convert_mapping(&actor_status, &vnode_mapping)?;
-
                 // Notify fragment mapping to frontend nodes.
                 let fragment_mapping = FragmentWorkerSlotMapping {
                     fragment_id: *fragment_id as FragmentId,
@@ -1434,23 +1521,6 @@ impl FragmentManager {
         Ok(())
     }
 
-    fn convert_mapping(
-        actor_status: &BTreeMap<ActorId, ActorStatus>,
-        vnode_mapping: &PbParallelUnitMapping,
-    ) -> MetaResult<PbWorkerSlotMapping> {
-        let parallel_unit_to_worker = actor_status
-            .values()
-            .map(|actor_status| {
-                let parallel_unit = actor_status.get_parallel_unit().unwrap();
-                (parallel_unit.id, parallel_unit.worker_node_id)
-            })
-            .collect();
-
-        Ok(ParallelUnitMapping::from_protobuf(vnode_mapping)
-            .to_worker_slot(&parallel_unit_to_worker)?
-            .to_protobuf())
-    }
-
     pub async fn table_node_actors(
         &self,
         table_ids: &HashSet<TableId>,
@@ -1503,10 +1573,12 @@ impl FragmentManager {
     pub async fn get_upstream_root_fragments(
         &self,
         upstream_table_ids: &HashSet<TableId>,
-    ) -> MetaResult<HashMap<TableId, Fragment>> {
+    ) -> MetaResult<(HashMap<TableId, Fragment>, HashMap<ActorId, u32>)> {
         let map = &self.core.read().await.table_fragments;
         let mut fragments = HashMap::new();
 
+        let mut actor_locations = HashMap::new();
+
         for &table_id in upstream_table_ids {
             let table_fragments = map
                 .get(&table_id)
@@ -1518,16 +1590,24 @@ impl FragmentManager {
                 // look for Source fragment if there's no MView fragment
                 fragments.insert(table_id, fragment);
             }
+
+            // todo: reduce memory usage
+            table_fragments
+                .actor_status
+                .iter()
+                .for_each(|(actor_id, status)| {
+                    actor_locations.insert(*actor_id, status.worker_id());
+                });
         }
 
-        Ok(fragments)
+        Ok((fragments, actor_locations))
     }
 
     /// Get the downstream `StreamTableScan` fragments of the specified MV.
     pub async fn get_downstream_fragments(
         &self,
         table_id: TableId,
-    ) -> MetaResult<Vec<(DispatchStrategy, Fragment)>> {
+    ) -> MetaResult<(Vec<(DispatchStrategy, Fragment)>, HashMap<ActorId, u32>)> {
         let map = &self.core.read().await.table_fragments;
 
         let table_fragments = map
@@ -1569,7 +1649,18 @@ impl FragmentManager {
 
         assert_eq!(downstream_dispatches.len(), fragments.len());
 
-        Ok(fragments)
+        let mut actor_locations = HashMap::new();
+
+        map.values().for_each(|table_fragments| {
+            table_fragments
+                .actor_status
+                .iter()
+                .for_each(|(actor_id, status)| {
+                    actor_locations.insert(*actor_id, status.worker_id());
+                });
+        });
+
+        Ok((fragments, actor_locations))
     }
 
     /// Get the `Materialize` fragment of the specified table.
diff --git a/src/meta/src/manager/catalog/mod.rs b/src/meta/src/manager/catalog/mod.rs
index 72024ff98711c..25c96dd4ec630 100644
--- a/src/meta/src/manager/catalog/mod.rs
+++ b/src/meta/src/manager/catalog/mod.rs
@@ -19,6 +19,7 @@ mod utils;
 
 use std::collections::{BTreeMap, HashMap, HashSet, VecDeque};
 use std::iter;
+use std::mem::take;
 use std::sync::Arc;
 
 use anyhow::{anyhow, Context};
@@ -30,6 +31,7 @@ use risingwave_common::catalog::{
     DEFAULT_SCHEMA_NAME, DEFAULT_SUPER_USER, DEFAULT_SUPER_USER_FOR_PG,
     DEFAULT_SUPER_USER_FOR_PG_ID, DEFAULT_SUPER_USER_ID, SYSTEM_SCHEMAS,
 };
+use risingwave_common::secret::LocalSecretManager;
 use risingwave_common::{bail, current_cluster_version, ensure};
 use risingwave_connector::source::{should_copy_to_format_encode_options, UPSTREAM_SOURCE_KEY};
 use risingwave_pb::catalog::subscription::PbSubscriptionState;
@@ -38,11 +40,12 @@ use risingwave_pb::catalog::{
     Comment, Connection, CreateType, Database, Function, Index, PbSource, PbStreamJobStatus,
     Schema, Secret, Sink, Source, StreamJobStatus, Subscription, Table, View,
 };
-use risingwave_pb::ddl_service::{alter_owner_request, alter_set_schema_request};
+use risingwave_pb::ddl_service::{alter_owner_request, alter_set_schema_request, TableJobType};
 use risingwave_pb::meta::subscribe_response::{Info, Operation};
 use risingwave_pb::user::grant_privilege::{Action, ActionWithGrantOption, Object};
 use risingwave_pb::user::update_user_request::UpdateField;
 use risingwave_pb::user::{GrantPrivilege, UserInfo};
+use tokio::sync::oneshot::Sender;
 use tokio::sync::{Mutex, MutexGuard};
 use user::*;
 
@@ -164,6 +167,92 @@ impl CatalogManagerCore {
         let user = UserManager::new(env.clone(), &database).await?;
         Ok(Self { database, user })
     }
+
+    pub(crate) fn register_finish_notifier(
+        &mut self,
+        id: TableId,
+        sender: Sender<MetaResult<NotificationVersion>>,
+    ) {
+        self.database
+            .creating_table_finish_notifier
+            .entry(id)
+            .or_default()
+            .push(sender);
+    }
+
+    pub(crate) fn streaming_job_is_finished(&mut self, job: &StreamingJob) -> MetaResult<bool> {
+        fn gen_err(job: &StreamingJob, name: &String) -> MetaError {
+            MetaError::catalog_id_not_found(
+                job.job_type_str(),
+                format!("{} may have been dropped/cancelled", name),
+            )
+        }
+        let (job_status, name) = match job {
+            StreamingJob::MaterializedView(table) | StreamingJob::Table(_, table, _) => (
+                self.database
+                    .tables
+                    .get(&table.id)
+                    .map(|table| table.stream_job_status),
+                &table.name,
+            ),
+            StreamingJob::Sink(sink, _) => (
+                self.database
+                    .sinks
+                    .get(&sink.id)
+                    .map(|sink| sink.stream_job_status),
+                &sink.name,
+            ),
+            StreamingJob::Index(index, _) => (
+                self.database
+                    .indexes
+                    .get(&index.id)
+                    .map(|index| index.stream_job_status),
+                &index.name,
+            ),
+            StreamingJob::Source(source) => {
+                return Ok(self.database.sources.contains_key(&source.id));
+            }
+        };
+
+        job_status
+            .map(|status| status == StreamJobStatus::Created as i32)
+            .or_else(|| {
+                if self
+                    .database
+                    .in_progress_creation_streaming_job
+                    .contains_key(&job.id())
+                {
+                    Some(false)
+                } else {
+                    None
+                }
+            })
+            .ok_or_else(|| gen_err(job, name))
+    }
+
+    pub(crate) fn notify_finish(&mut self, id: TableId, version: NotificationVersion) {
+        for tx in self
+            .database
+            .creating_table_finish_notifier
+            .remove(&id)
+            .into_iter()
+            .flatten()
+        {
+            let _ = tx.send(Ok(version));
+        }
+    }
+
+    pub(crate) fn notify_finish_failed(&mut self, err: &MetaError) {
+        for tx in take(&mut self.database.creating_table_finish_notifier)
+            .into_values()
+            .flatten()
+        {
+            let _ = tx.send(Err(err.clone()));
+        }
+        // Clear in progress creation streaming job. Note that background job is not tracked here, so that
+        // it won't affect background jobs.
+        self.database.in_progress_creation_streaming_job.clear();
+    }
 }
 
 impl CatalogManager {
@@ -178,9 +267,14 @@ impl CatalogManager {
         self.init_user().await?;
         self.init_database().await?;
         self.source_backward_compat_check().await?;
+        self.table_sink_catalog_update().await?;
         Ok(())
     }
 
+    pub async fn current_notification_version(&self) -> NotificationVersion {
+        self.env.notification_manager().current_version().await
+    }
+
     pub async fn get_catalog_core_guard(&self) -> MutexGuard<'_, CatalogManagerCore> {
         self.core.lock().await
     }
@@ -230,6 +324,37 @@ impl CatalogManager {
         commit_meta!(self, sources)?;
         Ok(())
     }
+
+    // Fill in the original_target_columns that wasn't written in the previous version for the table sink.
+    async fn table_sink_catalog_update(&self) -> MetaResult<()> {
+        let core = &mut *self.core.lock().await;
+        let mut sinks = BTreeMapTransaction::new(&mut core.database.sinks);
+        let tables = BTreeMapTransaction::new(&mut core.database.tables);
+        let legacy_sinks = sinks
+            .tree_ref()
+            .iter()
+            .filter(|(_, sink)| {
+                sink.target_table.is_some() && sink.original_target_columns.is_empty()
+            })
+            .map(|(_, sink)| sink.clone())
+            .collect_vec();
+
+        for mut sink in legacy_sinks {
+            let target_table = sink.target_table();
+            sink.original_target_columns
+                .clone_from(&tables.get(&target_table).unwrap().columns);
+            tracing::info!(
+                "updating sink {} target table columns {:?}",
+                sink.id,
+                sink.original_target_columns
+            );
+
+            sinks.insert(sink.id, sink);
+        }
+        commit_meta!(self, sinks)?;
+
+        Ok(())
+    }
 }
 
 // Database catalog related methods
@@ -487,7 +612,11 @@ impl CatalogManager {
         }
     }
 
-    pub async fn create_secret(&self, secret: Secret) -> MetaResult<NotificationVersion> {
+    pub async fn create_secret(
+        &self,
+        secret: Secret,
+        secret_plain_payload: Vec<u8>,
+    ) -> MetaResult<NotificationVersion> {
         let core = &mut *self.core.lock().await;
         let database_core = &mut core.database;
         let user_core = &mut core.user;
@@ -504,14 +633,25 @@ impl CatalogManager {
 
         let secret_id = secret.id;
         let mut secret_entry = BTreeMapTransaction::new(&mut database_core.secrets);
+
         secret_entry.insert(secret_id, secret.to_owned());
         commit_meta!(self, secret_entry)?;
 
         user_core.increase_ref(secret.owner);
 
+        // Notify the compute and frontend node plain secret
+        let mut secret_plain = secret;
+        secret_plain.value.clone_from(&secret_plain_payload);
+
+        LocalSecretManager::global().add_secret(secret_id, secret_plain_payload);
+        self.env
+            .notification_manager()
+            .notify_compute_without_version(Operation::Add, Info::Secret(secret_plain.clone()));
+
         let version = self
-            .notify_frontend(Operation::Add, Info::Secret(secret))
+            .notify_frontend(Operation::Add, Info::Secret(secret_plain))
             .await;
+
         Ok(version)
     }
 
@@ -521,21 +661,40 @@ impl CatalogManager {
         let user_core = &mut core.user;
         let mut secrets = BTreeMapTransaction::new(&mut database_core.secrets);
 
-        // todo: impl a ref count check for secret
-        // if secret is used by other relations, not found in the catalog or do not have the privilege to drop, return error
-        // else: commit the change and notify frontend
-
-        let secret = secrets
-            .remove(secret_id)
-            .ok_or_else(|| anyhow!("secret not found"))?;
-
-        commit_meta!(self, secrets)?;
-        user_core.decrease_ref(secret.owner);
+        match database_core.secret_ref_count.get(&secret_id) {
+            Some(ref_count) => {
+                let secret_name = secrets
+                    .get(&secret_id)
+                    .ok_or_else(|| MetaError::catalog_id_not_found("connection", secret_id))?
+                    .name
+                    .clone();
+                Err(MetaError::permission_denied(format!(
+                    "Fail to delete secret {} because {} other relation(s) depend on it",
+                    secret_name, ref_count
+                )))
+            }
+            None => {
+                let secret = secrets
+                    .remove(secret_id)
+                    .ok_or_else(|| anyhow!("secret not found"))?;
+
+                commit_meta!(self, secrets)?;
+                user_core.decrease_ref(secret.owner);
+
+                LocalSecretManager::global().remove_secret(secret.id);
+                self.env
+                    .notification_manager()
+                    .notify_compute_without_version(
+                        Operation::Delete,
+                        Info::Secret(secret.clone()),
+                    );
 
-        let version = self
-            .notify_frontend(Operation::Delete, Info::Secret(secret))
-            .await;
-        Ok(version)
+                let version = self
+                    .notify_frontend(Operation::Delete, Info::Secret(secret))
+                    .await;
+                Ok(version)
+            }
+        }
     }
 
     pub async fn create_connection(
@@ -853,7 +1012,7 @@ impl CatalogManager {
         database_core.check_relation_name_duplicated(&key)?;
 
         if database_core.has_in_progress_creation(&key) {
-            bail!("table is in creating procedure");
+            bail!("The table is being created");
         } else {
             database_core.mark_creating(&key);
             database_core.mark_creating_streaming_job(table.id, key);
@@ -1138,18 +1297,21 @@ impl CatalogManager {
         &self,
         mut stream_job: StreamingJob,
         internal_tables: Vec<Table>,
-    ) -> MetaResult<u64> {
+    ) -> MetaResult<()> {
         // 1. finish procedure.
         let mut creating_internal_table_ids = internal_tables.iter().map(|t| t.id).collect_vec();
 
         // Update the corresponding 'created_at' field.
         stream_job.mark_created();
 
-        let version = match stream_job {
+        let (version, table_id) = match stream_job {
             StreamingJob::MaterializedView(table) => {
                 creating_internal_table_ids.push(table.id);
-                self.finish_create_materialized_view_procedure(internal_tables, table)
-                    .await?
+                let table_id = table.id;
+                let version = self
+                    .finish_create_materialized_view_procedure(internal_tables, table)
+                    .await?;
+                (version, table_id)
             }
             StreamingJob::Sink(sink, target_table) => {
                 let sink_id = sink.id;
@@ -1160,30 +1322,45 @@ impl CatalogManager {
 
                 if let Some((table, source)) = target_table {
                     version = self
-                        .finish_replace_table_procedure(&source, &table, None, Some(sink_id), None)
+                        .finish_replace_table_procedure(
+                            &source,
+                            &table,
+                            None,
+                            Some(sink_id),
+                            None,
+                            vec![],
+                        )
                         .await?;
                 }
 
-                version
+                (version, sink_id)
             }
             StreamingJob::Table(source, table, ..) => {
                 creating_internal_table_ids.push(table.id);
-                if let Some(source) = source {
+                let table_id = table.id;
+                let version = if let Some(source) = source {
                     self.finish_create_table_procedure_with_source(source, table, internal_tables)
                         .await?
                 } else {
                     self.finish_create_table_procedure(internal_tables, table)
                         .await?
-                }
+                };
+                (version, table_id)
             }
             StreamingJob::Index(index, table) => {
                 creating_internal_table_ids.push(table.id);
-                self.finish_create_index_procedure(internal_tables, index, table)
-                    .await?
+                let table_id = table.id;
+                let version = self
+                    .finish_create_index_procedure(internal_tables, index, table)
+                    .await?;
+                (version, table_id)
             }
             StreamingJob::Source(source) => {
-                self.finish_create_source_procedure(source, internal_tables)
-                    .await?
+                let table_id = source.id;
+                let version = self
+                    .finish_create_source_procedure(source, internal_tables)
+                    .await?;
+                (version, table_id)
             }
         };
 
@@ -1191,7 +1368,10 @@ impl CatalogManager {
         self.unmark_creating_tables(&creating_internal_table_ids, false)
             .await;
 
-        Ok(version)
+        // 3. notify create streaming job finish
+        self.core.lock().await.notify_finish(table_id, version);
+
+        Ok(())
     }
 
     /// This is used for `CREATE TABLE`.
@@ -1295,66 +1475,72 @@ impl CatalogManager {
         table_id: TableId,
         internal_table_ids: Vec<TableId>,
     ) -> MetaResult<bool> {
-        let (table, internal_tables) = {
-            let core = &mut self.core.lock().await;
-            let database_core = &mut core.database;
-            let tables = &mut database_core.tables;
-            let Some(table) = tables.get(&table_id).cloned() else {
-                tracing::warn!(
-                    "table_id {} missing when attempting to cancel job, could be cleaned on recovery",
-                    table_id
-                );
-                return Ok(false);
-            };
-            let mut internal_tables = vec![];
-            for internal_table_id in &internal_table_ids {
-                if let Some(table) = tables.get(internal_table_id) {
-                    internal_tables.push(table.clone());
-                }
+        let core = &mut self.core.lock().await;
+        let database_core = &mut core.database;
+        let tables = &mut database_core.tables;
+        let Some(table) = tables.get(&table_id).cloned() else {
+            tracing::warn!(
+                "table_id {} missing when attempting to cancel job, could be cleaned on recovery",
+                table_id
+            );
+            return Ok(false);
+        };
+        let mut internal_tables = vec![];
+        for internal_table_id in &internal_table_ids {
+            if let Some(table) = tables.get(internal_table_id) {
+                internal_tables.push(table.clone());
             }
+        }
 
-            // `Unspecified` maps to Created state, due to backwards compatibility.
-            // `Created` states should not be cancelled.
-            if table
-                .get_stream_job_status()
-                .unwrap_or(StreamJobStatus::Created)
-                != StreamJobStatus::Creating
-            {
-                return Err(MetaError::invalid_parameter(format!(
-                    "table is not in creating state id={:#?}",
-                    table_id
-                )));
-            }
+        // `Unspecified` maps to Created state, due to backwards compatibility.
+        // `Created` states should not be cancelled.
+        if table
+            .get_stream_job_status()
+            .unwrap_or(StreamJobStatus::Created)
+            != StreamJobStatus::Creating
+        {
+            return Err(MetaError::invalid_parameter(format!(
+                "table is not in creating state id={:#?}",
+                table_id
+            )));
+        }
 
-            tracing::trace!("cleanup tables for {}", table.id);
-            let mut table_ids = vec![table.id];
-            table_ids.extend(internal_table_ids);
+        tracing::trace!("cleanup tables for {}", table.id);
+        let mut table_ids = vec![table.id];
+        table_ids.extend(internal_table_ids);
 
-            let tables = &mut database_core.tables;
-            let mut tables = BTreeMapTransaction::new(tables);
-            for table_id in table_ids {
-                let res = tables.remove(table_id);
-                assert!(res.is_some(), "table_id {} missing", table_id);
-            }
-            commit_meta!(self, tables)?;
-            (table, internal_tables)
-        };
+        let tables = &mut database_core.tables;
+        let mut tables = BTreeMapTransaction::new(tables);
+        for table_id in table_ids {
+            let res = tables.remove(table_id);
+            assert!(res.is_some(), "table_id {} missing", table_id);
+        }
+        commit_meta!(self, tables)?;
 
         {
-            let core = &mut self.core.lock().await;
-            {
-                let user_core = &mut core.user;
-                user_core.decrease_ref(table.owner);
-            }
+            let user_core = &mut core.user;
+            user_core.decrease_ref(table.owner);
+        }
 
-            {
-                let database_core = &mut core.database;
-                for &dependent_relation_id in &table.dependent_relations {
-                    database_core.decrease_relation_ref_count(dependent_relation_id);
-                }
+        {
+            let database_core = &mut core.database;
+            for &dependent_relation_id in &table.dependent_relations {
+                database_core.decrease_relation_ref_count(dependent_relation_id);
             }
         }
 
+        for tx in core
+            .database
+            .creating_table_finish_notifier
+            .remove(&table_id)
+            .into_iter()
+            .flatten()
+        {
+            let _ = tx.send(Err(MetaError::cancelled(format!(
+                "materialized view {table_id} has been cancelled"
+            ))));
+        }
+
         // FIXME(kwannoel): Propagate version to fe
         let _version = self
             .notify_frontend(
@@ -1371,6 +1557,7 @@ impl CatalogManager {
                 }),
             )
             .await;
+
         Ok(true)
     }
 
@@ -3557,7 +3744,7 @@ impl CatalogManager {
 
     /// This is used for `ALTER TABLE ADD/DROP COLUMN`.
     pub async fn start_replace_table_procedure(&self, stream_job: &StreamingJob) -> MetaResult<()> {
-        let StreamingJob::Table(source, table, ..) = stream_job else {
+        let StreamingJob::Table(source, table, job_type) = stream_job else {
             unreachable!("unexpected job: {stream_job:?}")
         };
         let core = &mut *self.core.lock().await;
@@ -3565,7 +3752,10 @@ impl CatalogManager {
         database_core.ensure_database_id(table.database_id)?;
         database_core.ensure_schema_id(table.schema_id)?;
 
-        assert!(table.dependent_relations.is_empty());
+        // general table streaming job should not have dependent relations
+        if matches!(job_type, TableJobType::General) {
+            assert!(table.dependent_relations.is_empty());
+        }
 
         let key = (table.database_id, table.schema_id, table.name.clone());
         let original_table = database_core
@@ -3594,7 +3784,7 @@ impl CatalogManager {
         }
     }
 
-    /// This is used for `ALTER TABLE ADD/DROP COLUMN`.
+    /// This is used for `ALTER TABLE ADD/DROP COLUMN` and `SINK INTO TABLE`.
     pub async fn finish_replace_table_procedure(
         &self,
         source: &Option<Source>,
@@ -3602,12 +3792,14 @@ impl CatalogManager {
         table_col_index_mapping: Option<ColIndexMapping>,
         creating_sink_id: Option<SinkId>,
         dropping_sink_id: Option<SinkId>,
+        updated_sink_ids: Vec<SinkId>,
     ) -> MetaResult<NotificationVersion> {
         let core = &mut *self.core.lock().await;
         let database_core = &mut core.database;
         let mut tables = BTreeMapTransaction::new(&mut database_core.tables);
         let mut sources = BTreeMapTransaction::new(&mut database_core.sources);
         let mut indexes = BTreeMapTransaction::new(&mut database_core.indexes);
+        let mut sinks = BTreeMapTransaction::new(&mut database_core.sinks);
         let key = (table.database_id, table.schema_id, table.name.clone());
 
         assert!(
@@ -3616,6 +3808,15 @@ impl CatalogManager {
             "table must exist and be in altering procedure"
         );
 
+        let original_table = tables.get(&table.id).unwrap();
+        let mut updated_sinks = vec![];
+        for sink_id in updated_sink_ids {
+            let mut sink = sinks.get_mut(sink_id).unwrap();
+            sink.original_target_columns
+                .clone_from(&original_table.columns);
+            updated_sinks.push(sink.clone());
+        }
+
         if let Some(source) = source {
             let source_key = (source.database_id, source.schema_id, source.name.clone());
             assert!(
@@ -3673,7 +3874,7 @@ impl CatalogManager {
 
         tables.insert(table.id, table.clone());
 
-        commit_meta!(self, tables, indexes, sources)?;
+        commit_meta!(self, tables, indexes, sources, sinks)?;
 
         // Group notification
         let version = self
@@ -3690,6 +3891,9 @@ impl CatalogManager {
                     .chain(updated_indexes.into_iter().map(|index| Relation {
                         relation_info: RelationInfo::Index(index).into(),
                     }))
+                    .chain(updated_sinks.into_iter().map(|sink| Relation {
+                        relation_info: RelationInfo::Sink(sink).into(),
+                    }))
                     .collect_vec(),
                 }),
             )
@@ -3720,8 +3924,6 @@ impl CatalogManager {
         let database_core = &mut core.database;
         let key = (table.database_id, table.schema_id, table.name.clone());
 
-        assert!(table.dependent_relations.is_empty());
-
         assert!(
             database_core.tables.contains_key(&table.id)
                 && database_core.has_in_progress_creation(&key),
@@ -4042,6 +4244,33 @@ impl CatalogManager {
         Ok(subscription.clone())
     }
 
+    pub async fn get_mv_depended_subscriptions(
+        &self,
+    ) -> MetaResult<HashMap<risingwave_common::catalog::TableId, HashMap<u32, u64>>> {
+        let guard = self.core.lock().await;
+        let mut map = HashMap::new();
+        for subscription in guard.database.subscriptions.values() {
+            map.entry(risingwave_common::catalog::TableId::from(
+                subscription.dependent_table_id,
+            ))
+            .or_insert(HashMap::new())
+            .insert(subscription.id, subscription.retention_seconds);
+        }
+
+        Ok(map)
+    }
+
+    pub async fn get_sinks(&self, sink_ids: &[SinkId]) -> Vec<Sink> {
+        let mut sinks = vec![];
+        let guard = self.core.lock().await;
+        for sink_id in sink_ids {
+            if let Some(sink) = guard.database.sinks.get(sink_id) {
+                sinks.push(sink.clone());
+            }
+        }
+        sinks
+    }
+
     pub async fn get_created_table_ids(&self) -> Vec<u32> {
         let guard = self.core.lock().await;
         guard
@@ -4618,4 +4847,36 @@ impl CatalogManager {
         }
         users_need_update
     }
+
+    pub async fn update_source_rate_limit_by_source_id(
+        &self,
+        source_id: SourceId,
+        rate_limit: Option<u32>,
+    ) -> MetaResult<()> {
+        let source_relation: PbSource;
+        {
+            let core = &mut *self.core.lock().await;
+            let database_core = &mut core.database;
+            let mut sources = BTreeMapTransaction::new(&mut database_core.sources);
+            let mut source = sources.get_mut(source_id);
+            let Some(source_catalog) = source.as_mut() else {
+                bail!("source {} not found", source_id)
+            };
+            source_relation = source_catalog.clone();
+            source_catalog.rate_limit = rate_limit;
+            commit_meta!(self, sources)?;
+        }
+
+        let _version = self
+            .notify_frontend(
+                Operation::Update,
+                Info::RelationGroup(RelationGroup {
+                    relations: vec![Relation {
+                        relation_info: RelationInfo::Source(source_relation).into(),
+                    }],
+                }),
+            )
+            .await;
+        Ok(())
+    }
 }
diff --git a/src/meta/src/manager/cluster.rs b/src/meta/src/manager/cluster.rs
index 1676e4ed3857e..93e50dec3706b 100644
--- a/src/meta/src/manager/cluster.rs
+++ b/src/meta/src/manager/cluster.rs
@@ -20,13 +20,12 @@ use std::sync::Arc;
 use std::time::{Duration, SystemTime};
 
 use itertools::Itertools;
-use risingwave_common::hash::ParallelUnitId;
 use risingwave_common::util::addr::HostAddr;
 use risingwave_common::util::resource_util::cpu::total_cpu_available;
 use risingwave_common::util::resource_util::memory::system_memory_available_bytes;
 use risingwave_common::RW_VERSION;
 use risingwave_pb::common::worker_node::{Property, State};
-use risingwave_pb::common::{HostAddress, ParallelUnit, WorkerNode, WorkerType};
+use risingwave_pb::common::{HostAddress, WorkerNode, WorkerType};
 use risingwave_pb::meta::add_worker_node_request::Property as AddNodeProperty;
 use risingwave_pb::meta::heartbeat_request;
 use risingwave_pb::meta::subscribe_response::{Info, Operation};
@@ -39,7 +38,8 @@ use tokio::task::JoinHandle;
 
 use crate::manager::{IdCategory, LocalNotification, MetaSrvEnv};
 use crate::model::{
-    InMemValTransaction, MetadataModel, ValTransaction, VarTransaction, Worker, INVALID_EXPIRE_AT,
+    ClusterId, InMemValTransaction, MetadataModel, ValTransaction, VarTransaction, Worker,
+    INVALID_EXPIRE_AT,
 };
 use crate::storage::{MetaStore, Transaction};
 use crate::{MetaError, MetaResult};
@@ -127,7 +127,7 @@ impl ClusterManager {
                     .unwrap_or_default();
             }
 
-            let old_worker_parallelism = worker.worker_node.parallel_units.len();
+            let old_worker_parallelism = worker.worker_node.parallelism();
             if old_worker_parallelism == new_worker_parallelism
                 && worker.worker_node.property == property
             {
@@ -144,13 +144,8 @@ impl ClusterManager {
                         old_worker_parallelism,
                         new_worker_parallelism
                     );
-                    let parallel_units = self
-                        .generate_cn_parallel_units(
-                            new_worker_parallelism - old_worker_parallelism,
-                            new_worker.worker_id(),
-                        )
-                        .await?;
-                    new_worker.worker_node.parallel_units.extend(parallel_units);
+
+                    new_worker.worker_node.parallelism = new_worker_parallelism as _;
                 }
                 Ordering::Greater => {
                     if !self.env.opts.disable_automatic_parallelism_control {
@@ -161,10 +156,8 @@ impl ClusterManager {
                             old_worker_parallelism,
                             new_worker_parallelism
                         );
-                        new_worker
-                            .worker_node
-                            .parallel_units
-                            .truncate(new_worker_parallelism)
+
+                        new_worker.worker_node.parallelism = new_worker_parallelism as _;
                     } else {
                         // Warn and keep the original parallelism if the worker registered with a
                         // smaller parallelism, entering compatibility mode.
@@ -210,25 +203,19 @@ impl ClusterManager {
             _ => None,
         };
 
-        // Generate parallel units.
-        let parallel_units = if r#type == WorkerType::ComputeNode {
-            self.generate_cn_parallel_units(new_worker_parallelism, worker_id)
-                .await?
-        } else {
-            vec![]
-        };
         // Construct worker.
         let worker_node = WorkerNode {
             id: worker_id,
             r#type: r#type as i32,
             host: Some(host_address.clone()),
             state: State::Starting as i32,
-            parallel_units,
+            parallelism: new_worker_parallelism as _,
             property,
             transactional_id,
             // resource doesn't need persist
             resource: None,
             started_at: None,
+            node_label: "".to_string(),
         };
 
         let mut worker = Worker::from_protobuf(worker_node.clone());
@@ -515,30 +502,13 @@ impl ClusterManager {
         }
     }
 
-    /// Generate `parallel_degree` parallel units.
-    async fn generate_cn_parallel_units(
-        &self,
-        parallel_degree: usize,
-        worker_id: WorkerId,
-    ) -> MetaResult<Vec<ParallelUnit>> {
-        let start_id = self
-            .env
-            .id_gen_manager()
-            .as_kv()
-            .generate_interval::<{ IdCategory::ParallelUnit }>(parallel_degree as u64)
-            .await? as ParallelUnitId;
-        let parallel_units = (start_id..start_id + parallel_degree as ParallelUnitId)
-            .map(|id| ParallelUnit {
-                id,
-                worker_node_id: worker_id,
-            })
-            .collect();
-        Ok(parallel_units)
-    }
-
     pub async fn get_worker_by_id(&self, worker_id: WorkerId) -> Option<Worker> {
         self.core.read().await.get_worker_by_id(worker_id)
     }
+
+    pub fn cluster_id(&self) -> &ClusterId {
+        self.env.cluster_id()
+    }
 }
 
 /// The cluster info used for scheduling a streaming job.
@@ -547,11 +517,18 @@ pub struct StreamingClusterInfo {
     /// All **active** compute nodes in the cluster.
     pub worker_nodes: HashMap<u32, WorkerNode>,
 
-    /// All parallel units of the **active** compute nodes in the cluster.
-    pub parallel_units: HashMap<ParallelUnitId, ParallelUnit>,
+    /// All unschedulable compute nodes in the cluster.
+    pub unschedulable_workers: HashSet<u32>,
+}
 
-    /// All unschedulable parallel units of compute nodes in the cluster.
-    pub unschedulable_parallel_units: HashMap<ParallelUnitId, ParallelUnit>,
+// Encapsulating the use of parallelism
+impl StreamingClusterInfo {
+    pub fn parallelism(&self) -> usize {
+        self.worker_nodes
+            .values()
+            .map(|worker| worker.parallelism as usize)
+            .sum()
+    }
 }
 
 pub struct ClusterManagerCore {
@@ -712,7 +689,7 @@ impl ClusterManagerCore {
             .collect()
     }
 
-    // List all parallel units on running nodes
+    // List all serving worker nodes
     pub fn list_serving_worker_node(&self, worker_state: Option<State>) -> Vec<WorkerNode> {
         self.list_worker_node(Some(WorkerType::ComputeNode), worker_state)
             .into_iter()
@@ -724,34 +701,24 @@ impl ClusterManagerCore {
     fn get_streaming_cluster_info(&self) -> StreamingClusterInfo {
         let mut streaming_worker_node = self.list_streaming_worker_node(Some(State::Running));
 
-        let unschedulable_worker_node = streaming_worker_node
+        let unschedulable_workers = streaming_worker_node
             .extract_if(|worker| {
                 worker
                     .property
                     .as_ref()
                     .map_or(false, |p| p.is_unschedulable)
             })
-            .collect_vec();
+            .map(|w| w.id)
+            .collect();
 
         let active_workers: HashMap<_, _> = streaming_worker_node
             .into_iter()
             .map(|w| (w.id, w))
             .collect();
 
-        let active_parallel_units = active_workers
-            .values()
-            .flat_map(|worker| worker.parallel_units.iter().map(|p| (p.id, p.clone())))
-            .collect();
-
-        let unschedulable_parallel_units = unschedulable_worker_node
-            .iter()
-            .flat_map(|worker| worker.parallel_units.iter().map(|p| (p.id, p.clone())))
-            .collect();
-
         StreamingClusterInfo {
             worker_nodes: active_workers,
-            parallel_units: active_parallel_units,
-            unschedulable_parallel_units,
+            unschedulable_workers,
         }
     }
 
@@ -796,7 +763,6 @@ fn meta_node_info(host: &str, started_at: Option<u64>) -> WorkerNode {
             .map(HostAddr::to_protobuf)
             .ok(),
         state: State::Running as _,
-        parallel_units: vec![],
         property: None,
         transactional_id: None,
         resource: Some(risingwave_pb::common::worker_node::Resource {
@@ -805,6 +771,8 @@ fn meta_node_info(host: &str, started_at: Option<u64>) -> WorkerNode {
             total_cpu_cores: total_cpu_available() as _,
         }),
         started_at,
+        parallelism: 0,
+        node_label: "".to_string(),
     }
 }
 
@@ -847,7 +815,7 @@ mod tests {
             worker_nodes.push(worker_node);
         }
 
-        // Since no worker is active, the parallel unit count should be 0.
+        // Since no worker is active, the parallelism should be 0.
         assert_cluster_manager(&cluster_manager, 0).await;
 
         for worker_node in worker_nodes {
@@ -885,7 +853,7 @@ mod tests {
             )
             .await
             .unwrap();
-        assert_eq!(worker_node.parallel_units.len(), fake_parallelism + 4);
+        assert_eq!(worker_node.parallelism(), fake_parallelism + 4);
         assert_cluster_manager(&cluster_manager, parallel_count + 4).await;
 
         // re-register existing worker node with smaller parallelism.
@@ -909,11 +877,11 @@ mod tests {
             .unwrap();
 
         if !env.opts.disable_automatic_parallelism_control {
-            assert_eq!(worker_node.parallel_units.len(), fake_parallelism - 2);
+            assert_eq!(worker_node.parallelism(), fake_parallelism - 2);
             assert_cluster_manager(&cluster_manager, parallel_count - 2).await;
         } else {
             // compatibility mode
-            assert_eq!(worker_node.parallel_units.len(), fake_parallelism + 4);
+            assert_eq!(worker_node.parallelism(), fake_parallelism + 4);
             assert_cluster_manager(&cluster_manager, parallel_count + 4).await;
         }
 
@@ -979,13 +947,13 @@ mod tests {
     }
 
     async fn assert_cluster_manager(cluster_manager: &ClusterManager, parallel_count: usize) {
-        let parallel_units = cluster_manager
+        let parallelism: usize = cluster_manager
             .list_active_serving_compute_nodes()
             .await
             .into_iter()
-            .flat_map(|w| w.parallel_units)
-            .collect_vec();
-        assert_eq!(parallel_units.len(), parallel_count);
+            .map(|w| w.parallelism as usize)
+            .sum();
+        assert_eq!(parallelism, parallel_count);
     }
 
     // This test takes seconds because the TTL is measured in seconds.
diff --git a/src/meta/src/manager/diagnose.rs b/src/meta/src/manager/diagnose.rs
index 06c76c47c5daa..cb5f005829e80 100644
--- a/src/meta/src/manager/diagnose.rs
+++ b/src/meta/src/manager/diagnose.rs
@@ -13,7 +13,7 @@
 // limitations under the License.
 
 use std::cmp::{Ordering, Reverse};
-use std::collections::BinaryHeap;
+use std::collections::{BTreeMap, BinaryHeap};
 use std::fmt::Write;
 use std::sync::Arc;
 
@@ -21,18 +21,22 @@ use itertools::Itertools;
 use prometheus_http_query::response::Data::Vector;
 use risingwave_common::types::Timestamptz;
 use risingwave_common::util::StackTraceResponseExt;
+use risingwave_hummock_sdk::level::Level;
+use risingwave_meta_model_v2::table::TableType;
 use risingwave_pb::common::WorkerType;
-use risingwave_pb::hummock::Level;
 use risingwave_pb::meta::event_log::Event;
 use risingwave_pb::meta::EventLog;
 use risingwave_pb::monitor_service::StackTraceResponse;
 use risingwave_rpc_client::ComputeClientPool;
+use risingwave_sqlparser::ast::{CompatibleSourceSchema, Statement, Value};
+use risingwave_sqlparser::parser::Parser;
 use serde_json::json;
 use thiserror_ext::AsReport;
 
 use crate::hummock::HummockManagerRef;
 use crate::manager::event_log::EventLogMangerRef;
-use crate::manager::MetadataManager;
+use crate::manager::{MetadataManager, MetadataManagerV2};
+use crate::MetaResult;
 
 pub type DiagnoseCommandRef = Arc<DiagnoseCommand>;
 
@@ -88,7 +92,15 @@ impl DiagnoseCommand {
     async fn write_catalog(&self, s: &mut String) {
         match &self.metadata_manager {
             MetadataManager::V1(_) => self.write_catalog_v1(s).await,
-            MetadataManager::V2(_) => self.write_catalog_v2(s).await,
+            MetadataManager::V2(mgr) => {
+                self.write_catalog_v2(s).await;
+                let _ = self.write_table_definition(mgr, s).await.inspect_err(|e| {
+                    tracing::warn!(
+                        error = e.to_report_string(),
+                        "failed to display table definition"
+                    )
+                });
+            }
         }
     }
 
@@ -214,7 +226,7 @@ impl DiagnoseCommand {
                 &mut row,
                 worker_node.get_state().ok().map(|s| s.as_str_name()),
             );
-            row.add_cell(worker_node.parallel_units.len().into());
+            row.add_cell(worker_node.parallelism().into());
             try_add_cell(
                 &mut row,
                 worker_node.property.as_ref().map(|p| p.is_streaming),
@@ -458,7 +470,6 @@ impl DiagnoseCommand {
 
         let top_k = 10;
         let mut top_tombstone_delete_sst = BinaryHeap::with_capacity(top_k);
-        let mut top_range_delete_sst = BinaryHeap::with_capacity(top_k);
         for compaction_group in version.levels.values() {
             let mut visit_level = |level: &Level| {
                 sst_num += level.table_infos.len();
@@ -474,20 +485,9 @@ impl DiagnoseCommand {
                         delete_ratio: tombstone_delete_ratio,
                     };
                     top_k_sstables(top_k, &mut top_tombstone_delete_sst, e);
-
-                    let range_delete_ratio =
-                        sst.range_tombstone_count * 10000 / sst.total_key_count;
-                    let e = SstableSort {
-                        compaction_group_id: compaction_group.group_id,
-                        sst_id: sst.sst_id,
-                        delete_ratio: range_delete_ratio,
-                    };
-                    top_k_sstables(top_k, &mut top_range_delete_sst, e);
                 }
             };
-            let Some(ref l0) = compaction_group.l0 else {
-                continue;
-            };
+            let l0 = &compaction_group.l0;
             // FIXME: why chaining levels iter leads to segmentation fault?
             for level in &l0.sub_levels {
                 visit_level(level);
@@ -522,8 +522,6 @@ impl DiagnoseCommand {
         let _ = writeln!(s, "top tombstone delete ratio");
         let _ = writeln!(s, "{}", format_table(top_tombstone_delete_sst));
         let _ = writeln!(s);
-        let _ = writeln!(s, "top range delete ratio");
-        let _ = writeln!(s, "{}", format_table(top_range_delete_sst));
 
         let _ = writeln!(s);
         self.write_storage_prometheus(s).await;
@@ -667,7 +665,7 @@ impl DiagnoseCommand {
 
         let mut all = StackTraceResponse::default();
 
-        let compute_clients = ComputeClientPool::default();
+        let compute_clients = ComputeClientPool::adhoc();
         for worker_node in &worker_nodes {
             if let Ok(client) = compute_clients.get(worker_node).await
                 && let Ok(result) = client.stack_trace().await
@@ -678,6 +676,81 @@ impl DiagnoseCommand {
 
         write!(s, "{}", all.output()).unwrap();
     }
+
+    async fn write_table_definition(
+        &self,
+        mgr: &MetadataManagerV2,
+        s: &mut String,
+    ) -> MetaResult<()> {
+        let sources = mgr
+            .catalog_controller
+            .list_sources()
+            .await?
+            .into_iter()
+            .map(|s| (s.id, (s.name, s.schema_id, s.definition)))
+            .collect::<BTreeMap<_, _>>();
+        let tables = mgr
+            .catalog_controller
+            .list_tables_by_type(TableType::Table)
+            .await?
+            .into_iter()
+            .map(|t| (t.id, (t.name, t.schema_id, t.definition)))
+            .collect::<BTreeMap<_, _>>();
+        let mvs = mgr
+            .catalog_controller
+            .list_tables_by_type(TableType::MaterializedView)
+            .await?
+            .into_iter()
+            .map(|t| (t.id, (t.name, t.schema_id, t.definition)))
+            .collect::<BTreeMap<_, _>>();
+        let indexes = mgr
+            .catalog_controller
+            .list_tables_by_type(TableType::Index)
+            .await?
+            .into_iter()
+            .map(|t| (t.id, (t.name, t.schema_id, t.definition)))
+            .collect::<BTreeMap<_, _>>();
+        let sinks = mgr
+            .catalog_controller
+            .list_sinks()
+            .await?
+            .into_iter()
+            .map(|s| (s.id, (s.name, s.schema_id, s.definition)))
+            .collect::<BTreeMap<_, _>>();
+        let catalogs = [
+            ("SOURCE", sources),
+            ("TABLE", tables),
+            ("MATERIALIZED VIEW", mvs),
+            ("INDEX", indexes),
+            ("SINK", sinks),
+        ];
+        for (title, items) in catalogs {
+            use comfy_table::{Row, Table};
+            let mut table = Table::new();
+            table.set_header({
+                let mut row = Row::new();
+                row.add_cell("id".into());
+                row.add_cell("name".into());
+                row.add_cell("schema_id".into());
+                row.add_cell("definition".into());
+                row
+            });
+            for (id, (name, schema_id, definition)) in items {
+                let mut row = Row::new();
+                let may_redact =
+                    redact_all_sql_options(&definition).unwrap_or_else(|| "[REDACTED]".into());
+                row.add_cell(id.into());
+                row.add_cell(name.into());
+                row.add_cell(schema_id.into());
+                row.add_cell(may_redact.into());
+                table.add_row(row);
+            }
+            let _ = writeln!(s);
+            let _ = writeln!(s, "{title}");
+            let _ = writeln!(s, "{table}");
+        }
+        Ok(())
+    }
 }
 
 #[cfg_attr(coverage, coverage(off))]
@@ -696,3 +769,52 @@ fn try_add_cell<T: Into<comfy_table::Cell>>(row: &mut comfy_table::Row, t: Optio
 fn merge_prometheus_selector<'a>(selectors: impl IntoIterator<Item = &'a str>) -> String {
     selectors.into_iter().filter(|s| !s.is_empty()).join(",")
 }
+
+fn redact_all_sql_options(sql: &str) -> Option<String> {
+    let Ok(mut statements) = Parser::parse_sql(sql) else {
+        return None;
+    };
+    let mut redacted = String::new();
+    for statement in &mut statements {
+        let options = match statement {
+            Statement::CreateTable {
+                with_options,
+                source_schema,
+                ..
+            } => {
+                let connector_schema = match source_schema {
+                    Some(CompatibleSourceSchema::V2(cs)) => Some(&mut cs.row_options),
+                    _ => None,
+                };
+                (Some(with_options), connector_schema)
+            }
+            Statement::CreateSource { stmt } => {
+                let connector_schema = match &mut stmt.source_schema {
+                    CompatibleSourceSchema::V2(cs) => Some(&mut cs.row_options),
+                    _ => None,
+                };
+                (Some(&mut stmt.with_properties.0), connector_schema)
+            }
+            Statement::CreateSink { stmt } => {
+                let connector_schema = match &mut stmt.sink_schema {
+                    Some(cs) => Some(&mut cs.row_options),
+                    _ => None,
+                };
+                (Some(&mut stmt.with_properties.0), connector_schema)
+            }
+            _ => (None, None),
+        };
+        if let Some(options) = options.0 {
+            for option in options {
+                option.value = Value::SingleQuotedString("[REDACTED]".into());
+            }
+        }
+        if let Some(options) = options.1 {
+            for option in options {
+                option.value = Value::SingleQuotedString("[REDACTED]".into());
+            }
+        }
+        writeln!(&mut redacted, "{statement}").unwrap();
+    }
+    Some(redacted)
+}
diff --git a/src/meta/src/manager/env.rs b/src/meta/src/manager/env.rs
index e10d64a656328..22c7df751ec78 100644
--- a/src/meta/src/manager/env.rs
+++ b/src/meta/src/manager/env.rs
@@ -170,6 +170,7 @@ pub struct MetaOpts {
     /// Interval of hummock version checkpoint.
     pub hummock_version_checkpoint_interval_sec: u64,
     pub enable_hummock_data_archive: bool,
+    pub hummock_time_travel_snapshot_interval: u64,
     /// The minimum delta log number a new checkpoint should compact, otherwise the checkpoint
     /// attempt is rejected. Greater value reduces object store IO, meanwhile it results in
     /// more loss of in memory `HummockVersionCheckpoint::stale_objects` state when meta node is
@@ -283,7 +284,9 @@ pub struct MetaOpts {
     pub compact_task_table_size_partition_threshold_high: u64,
 
     // The private key for the secret store, used when the secret is stored in the meta.
-    pub secret_store_private_key: Vec<u8>,
+    pub secret_store_private_key: Option<Vec<u8>>,
+    /// The path of the temp secret file directory.
+    pub temp_secret_file_dir: String,
 
     pub table_info_statistic_history_times: usize,
 }
@@ -305,6 +308,7 @@ impl MetaOpts {
             vacuum_spin_interval_ms: 0,
             hummock_version_checkpoint_interval_sec: 30,
             enable_hummock_data_archive: false,
+            hummock_time_travel_snapshot_interval: 0,
             min_delta_log_num_for_hummock_version_checkpoint: 1,
             min_sst_retention_time_sec: 3600 * 24 * 7,
             full_gc_interval_sec: 3600 * 24 * 7,
@@ -346,7 +350,8 @@ impl MetaOpts {
             object_store_config: ObjectStoreConfig::default(),
             max_trivial_move_task_count_per_loop: 256,
             max_get_task_probe_times: 5,
-            secret_store_private_key: "demo-secret-private-key".as_bytes().to_vec(),
+            secret_store_private_key: Some("0123456789abcdef".as_bytes().to_vec()),
+            temp_secret_file_dir: "./secrets".to_string(),
             table_info_statistic_history_times: 240,
         }
     }
@@ -379,7 +384,7 @@ impl MetaSrvEnv {
         meta_store_impl: MetaStoreImpl,
     ) -> MetaResult<Self> {
         let idle_manager = Arc::new(IdleManager::new(opts.max_idle_ms));
-        let stream_client_pool = Arc::new(StreamClientPool::default());
+        let stream_client_pool = Arc::new(StreamClientPool::new(1)); // typically no need for plural clients
         let event_log_manager = Arc::new(start_event_log_manager(
             opts.event_log_enabled,
             opts.event_log_channel_max_size,
diff --git a/src/meta/src/manager/id.rs b/src/meta/src/manager/id.rs
index 023483116fdc8..95aa4f1935b58 100644
--- a/src/meta/src/manager/id.rs
+++ b/src/meta/src/manager/id.rs
@@ -128,7 +128,6 @@ pub mod IdCategory {
     pub const Actor: IdCategoryType = 6;
     pub const Backup: IdCategoryType = 7;
     pub const HummockSstableId: IdCategoryType = 8;
-    pub const ParallelUnit: IdCategoryType = 9;
     pub const _Source: IdCategoryType = 10;
     pub const HummockCompactionTask: IdCategoryType = 11;
     pub const User: IdCategoryType = 12;
@@ -159,7 +158,6 @@ pub struct IdGeneratorManager {
     backup: Arc<StoredIdGenerator>,
     hummock_ss_table_id: Arc<StoredIdGenerator>,
     hummock_compaction_task: Arc<StoredIdGenerator>,
-    parallel_unit: Arc<StoredIdGenerator>,
     compaction_group: Arc<StoredIdGenerator>,
     connection: Arc<StoredIdGenerator>,
     secret: Arc<StoredIdGenerator>,
@@ -198,9 +196,6 @@ impl IdGeneratorManager {
                 StoredIdGenerator::new(meta_store.clone(), "hummock_compaction_task", Some(1))
                     .await,
             ),
-            parallel_unit: Arc::new(
-                StoredIdGenerator::new(meta_store.clone(), "parallel_unit", None).await,
-            ),
             compaction_group: Arc::new(
                 StoredIdGenerator::new(
                     meta_store.clone(),
@@ -230,7 +225,6 @@ impl IdGeneratorManager {
             IdCategory::Backup => &self.backup,
             IdCategory::Worker => &self.worker,
             IdCategory::HummockSstableId => &self.hummock_ss_table_id,
-            IdCategory::ParallelUnit => &self.parallel_unit,
             IdCategory::HummockCompactionTask => &self.hummock_compaction_task,
             IdCategory::CompactionGroup => &self.compaction_group,
             IdCategory::Connection => &self.connection,
diff --git a/src/meta/src/manager/idle.rs b/src/meta/src/manager/idle.rs
index afa9fbf860932..431575e665b88 100644
--- a/src/meta/src/manager/idle.rs
+++ b/src/meta/src/manager/idle.rs
@@ -16,7 +16,7 @@ use std::sync::atomic::{AtomicU64, Ordering};
 use std::sync::Arc;
 use std::time::{Duration, Instant};
 
-use tokio::sync::oneshot::Sender;
+use risingwave_common::util::tokio_util::sync::CancellationToken;
 use tokio::task::JoinHandle;
 
 /// `IdleManager` keeps track of latest activity and report whether the meta service has been
@@ -77,24 +77,17 @@ impl IdleManager {
     pub fn start_idle_checker(
         idle_manager: IdleManagerRef,
         check_interval: Duration,
-        idle_send: tokio::sync::oneshot::Sender<()>,
-    ) -> (JoinHandle<()>, Sender<()>) {
+        shutdown: CancellationToken,
+    ) -> JoinHandle<()> {
         let dur = idle_manager.get_config_max_idle();
         if !dur.is_zero() {
             tracing::warn!("--dangerous-max-idle-secs is set. The meta server will be automatically stopped after idle for {:?}.", dur)
         }
 
-        let (shutdown_tx, mut shutdown_rx) = tokio::sync::oneshot::channel();
-        let join_handle = tokio::spawn(async move {
+        tokio::spawn(async move {
             let mut min_interval = tokio::time::interval(check_interval);
             loop {
-                tokio::select! {
-                    _ = min_interval.tick() => {},
-                    _ = &mut shutdown_rx => {
-                        tracing::info!("Idle checker is stopped");
-                        return;
-                    }
-                }
+                min_interval.tick().await;
                 if idle_manager.is_exceeding_max_idle() {
                     break;
                 }
@@ -104,9 +97,9 @@ impl IdleManager {
                 idle_manager.get_config_max_idle()
             );
             tracing::warn!("Idle checker is shutting down the server");
-            let _ = idle_send.send(());
-        });
-        (join_handle, shutdown_tx)
+
+            shutdown.cancel();
+        })
     }
 }
 
diff --git a/src/meta/src/manager/metadata.rs b/src/meta/src/manager/metadata.rs
index 241a47941755b..f634024a3a605 100644
--- a/src/meta/src/manager/metadata.rs
+++ b/src/meta/src/manager/metadata.rs
@@ -16,10 +16,11 @@ use std::collections::{BTreeMap, HashMap, HashSet};
 use std::pin::pin;
 use std::time::Duration;
 
+use anyhow::anyhow;
 use futures::future::{select, Either};
 use risingwave_common::catalog::{TableId, TableOption};
-use risingwave_meta_model_v2::SourceId;
-use risingwave_pb::catalog::{PbSource, PbTable};
+use risingwave_meta_model_v2::{ObjectId, SourceId};
+use risingwave_pb::catalog::{PbSink, PbSource, PbTable};
 use risingwave_pb::common::worker_node::{PbResource, State};
 use risingwave_pb::common::{HostAddress, PbWorkerNode, PbWorkerType, WorkerNode, WorkerType};
 use risingwave_pb::meta::add_worker_node_request::Property as AddNodeProperty;
@@ -27,7 +28,8 @@ use risingwave_pb::meta::table_fragments::{ActorStatus, Fragment, PbFragment};
 use risingwave_pb::stream_plan::{PbDispatchStrategy, StreamActor};
 use risingwave_pb::stream_service::BuildActorInfo;
 use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver};
-use tokio::time::sleep;
+use tokio::sync::oneshot;
+use tokio::time::{sleep, Instant};
 use tracing::warn;
 
 use crate::barrier::Reschedule;
@@ -35,12 +37,14 @@ use crate::controller::catalog::CatalogControllerRef;
 use crate::controller::cluster::{ClusterControllerRef, WorkerExtraInfo};
 use crate::manager::{
     CatalogManagerRef, ClusterManagerRef, FragmentManagerRef, LocalNotification,
-    StreamingClusterInfo, WorkerId,
+    NotificationVersion, StreamingClusterInfo, StreamingJob, WorkerId,
+};
+use crate::model::{
+    ActorId, ClusterId, FragmentId, MetadataModel, TableFragments, TableParallelism,
 };
-use crate::model::{ActorId, FragmentId, MetadataModel, TableFragments, TableParallelism};
 use crate::stream::{to_build_actor_info, SplitAssignment};
 use crate::telemetry::MetaTelemetryJobDesc;
-use crate::MetaResult;
+use crate::{MetaError, MetaResult};
 
 #[derive(Clone)]
 pub enum MetadataManager {
@@ -100,14 +104,21 @@ impl ActiveStreamingWorkerNodes {
     pub(crate) async fn wait_changed(
         &mut self,
         verbose_internal: Duration,
+        verbose_timeout: Duration,
         verbose_fn: impl Fn(&Self),
-    ) -> ActiveStreamingWorkerChange {
+    ) -> Option<ActiveStreamingWorkerChange> {
+        let start = Instant::now();
         loop {
             if let Either::Left((change, _)) =
                 select(pin!(self.changed()), pin!(sleep(verbose_internal))).await
             {
-                break change;
+                break Some(change);
+            }
+
+            if start.elapsed() > verbose_timeout {
+                break None;
             }
+
             verbose_fn(self)
         }
     }
@@ -455,7 +466,7 @@ impl MetadataManager {
     pub async fn get_upstream_root_fragments(
         &self,
         upstream_table_ids: &HashSet<TableId>,
-    ) -> MetaResult<HashMap<TableId, Fragment>> {
+    ) -> MetaResult<(HashMap<TableId, Fragment>, HashMap<ActorId, u32>)> {
         match self {
             MetadataManager::V1(mgr) => {
                 mgr.fragment_manager
@@ -463,7 +474,7 @@ impl MetadataManager {
                     .await
             }
             MetadataManager::V2(mgr) => {
-                let upstream_root_fragments = mgr
+                let (upstream_root_fragments, actors) = mgr
                     .catalog_controller
                     .get_upstream_root_fragments(
                         upstream_table_ids
@@ -472,10 +483,19 @@ impl MetadataManager {
                             .collect(),
                     )
                     .await?;
-                Ok(upstream_root_fragments
+
+                let actors = actors
                     .into_iter()
-                    .map(|(id, fragment)| ((id as u32).into(), fragment))
-                    .collect())
+                    .map(|(actor, worker)| (actor as u32, worker as u32))
+                    .collect();
+
+                Ok((
+                    upstream_root_fragments
+                        .into_iter()
+                        .map(|(id, fragment)| ((id as u32).into(), fragment))
+                        .collect(),
+                    actors,
+                ))
             }
         }
     }
@@ -534,10 +554,21 @@ impl MetadataManager {
         }
     }
 
+    pub async fn get_sink_catalog_by_ids(&self, ids: &[u32]) -> MetaResult<Vec<PbSink>> {
+        match &self {
+            MetadataManager::V1(mgr) => Ok(mgr.catalog_manager.get_sinks(ids).await),
+            MetadataManager::V2(mgr) => {
+                mgr.catalog_controller
+                    .get_sink_by_ids(ids.iter().map(|id| *id as _).collect())
+                    .await
+            }
+        }
+    }
+
     pub async fn get_downstream_chain_fragments(
         &self,
         job_id: u32,
-    ) -> MetaResult<Vec<(PbDispatchStrategy, PbFragment)>> {
+    ) -> MetaResult<(Vec<(PbDispatchStrategy, PbFragment)>, HashMap<ActorId, u32>)> {
         match &self {
             MetadataManager::V1(mgr) => {
                 mgr.fragment_manager
@@ -545,9 +576,17 @@ impl MetadataManager {
                     .await
             }
             MetadataManager::V2(mgr) => {
-                mgr.catalog_controller
+                let (fragments, actors) = mgr
+                    .catalog_controller
                     .get_downstream_chain_fragments(job_id as _)
-                    .await
+                    .await?;
+
+                let actors = actors
+                    .into_iter()
+                    .map(|(actor, worker)| (actor as u32, worker as u32))
+                    .collect();
+
+                Ok((fragments, actors))
             }
         }
     }
@@ -763,6 +802,9 @@ impl MetadataManager {
     ) -> MetaResult<HashMap<FragmentId, Vec<ActorId>>> {
         match self {
             MetadataManager::V1(mgr) => {
+                mgr.catalog_manager
+                    .update_source_rate_limit_by_source_id(source_id as u32, rate_limit)
+                    .await?;
                 mgr.fragment_manager
                     .update_source_rate_limit_by_source_id(source_id, rate_limit)
                     .await
@@ -822,11 +864,88 @@ impl MetadataManager {
         }
     }
 
-    #[expect(clippy::unused_async)]
     pub async fn get_mv_depended_subscriptions(
         &self,
     ) -> MetaResult<HashMap<TableId, HashMap<u32, u64>>> {
-        // TODO(subscription): support the correct logic when supporting L0 log store subscriptions
-        Ok(HashMap::new())
+        match self {
+            MetadataManager::V1(mgr) => mgr.catalog_manager.get_mv_depended_subscriptions().await,
+            MetadataManager::V2(mgr) => {
+                mgr.catalog_controller.get_mv_depended_subscriptions().await
+            }
+        }
+    }
+
+    pub fn cluster_id(&self) -> &ClusterId {
+        match self {
+            MetadataManager::V1(mgr) => mgr.cluster_manager.cluster_id(),
+            MetadataManager::V2(mgr) => mgr.cluster_controller.cluster_id(),
+        }
+    }
+}
+
+impl MetadataManager {
+    pub(crate) async fn wait_streaming_job_finished(
+        &self,
+        job: &StreamingJob,
+    ) -> MetaResult<NotificationVersion> {
+        match self {
+            MetadataManager::V1(mgr) => mgr.wait_streaming_job_finished(job).await,
+            MetadataManager::V2(mgr) => mgr.wait_streaming_job_finished(job.id() as _).await,
+        }
+    }
+
+    pub(crate) async fn notify_finish_failed(&self, err: &MetaError) {
+        match self {
+            MetadataManager::V1(mgr) => {
+                mgr.notify_finish_failed(err).await;
+            }
+            MetadataManager::V2(mgr) => {
+                mgr.notify_finish_failed(err).await;
+            }
+        }
+    }
+}
+
+impl MetadataManagerV2 {
+    pub(crate) async fn wait_streaming_job_finished(
+        &self,
+        id: ObjectId,
+    ) -> MetaResult<NotificationVersion> {
+        let mut mgr = self.catalog_controller.get_inner_write_guard().await;
+        if mgr.streaming_job_is_finished(id).await? {
+            return Ok(self.catalog_controller.current_notification_version().await);
+        }
+        let (tx, rx) = oneshot::channel();
+
+        mgr.register_finish_notifier(id, tx);
+        drop(mgr);
+        rx.await.map_err(|e| anyhow!(e))?
+    }
+
+    pub(crate) async fn notify_finish_failed(&self, err: &MetaError) {
+        let mut mgr = self.catalog_controller.get_inner_write_guard().await;
+        mgr.notify_finish_failed(err);
+    }
+}
+
+impl MetadataManagerV1 {
+    pub(crate) async fn wait_streaming_job_finished(
+        &self,
+        job: &StreamingJob,
+    ) -> MetaResult<NotificationVersion> {
+        let mut mgr = self.catalog_manager.get_catalog_core_guard().await;
+        if mgr.streaming_job_is_finished(job)? {
+            return Ok(self.catalog_manager.current_notification_version().await);
+        }
+        let (tx, rx) = oneshot::channel();
+
+        mgr.register_finish_notifier(job.id(), tx);
+        drop(mgr);
+        rx.await.map_err(|e| anyhow!(e))?
+    }
+
+    pub(crate) async fn notify_finish_failed(&self, err: &MetaError) {
+        let mut mgr = self.catalog_manager.get_catalog_core_guard().await;
+        mgr.notify_finish_failed(err);
     }
 }
diff --git a/src/meta/src/manager/sink_coordination/coordinator_worker.rs b/src/meta/src/manager/sink_coordination/coordinator_worker.rs
index 3ea9e8819214f..8409e714852c2 100644
--- a/src/meta/src/manager/sink_coordination/coordinator_worker.rs
+++ b/src/meta/src/manager/sink_coordination/coordinator_worker.rs
@@ -19,7 +19,7 @@ use anyhow::anyhow;
 use futures::future::{select, Either};
 use futures::stream::FuturesUnordered;
 use futures::{StreamExt, TryStreamExt};
-use risingwave_common::buffer::Bitmap;
+use risingwave_common::bitmap::Bitmap;
 use risingwave_common::hash::{VirtualNode, VnodeBitmapExt};
 use risingwave_connector::dispatch_sink;
 use risingwave_connector::sink::{build_sink, Sink, SinkCommitCoordinator, SinkParam};
@@ -153,7 +153,7 @@ impl CoordinatorWorker {
                 "one sink writer stream reaches the end before initialize"
             )),
             Either::Right((Some(Err(e)), _)) => {
-                Err(anyhow!(e).context("unable to poll from one sink writer stream"))
+                Err(anyhow!(e).context("unable to poll one sink writer stream"))
             }
             Either::Right((None, _)) => unreachable!("request_streams must not be empty"),
         }
@@ -267,8 +267,9 @@ impl CoordinatorWorker {
                         ));
                     }
                     Err(e) => {
-                        return Err(anyhow!(e)
-                            .context("failed to poll from one of the writer request streams"));
+                        return Err(
+                            anyhow!(e).context("failed to poll one of the writer request streams")
+                        );
                     }
                 },
                 Either::Right((None, _)) => {
diff --git a/src/meta/src/manager/sink_coordination/manager.rs b/src/meta/src/manager/sink_coordination/manager.rs
index a9327b4613a35..fd2b986be28e7 100644
--- a/src/meta/src/manager/sink_coordination/manager.rs
+++ b/src/meta/src/manager/sink_coordination/manager.rs
@@ -19,7 +19,7 @@ use std::pin::pin;
 use futures::future::{select, BoxFuture, Either};
 use futures::stream::FuturesUnordered;
 use futures::{FutureExt, Stream, StreamExt, TryStreamExt};
-use risingwave_common::buffer::Bitmap;
+use risingwave_common::bitmap::Bitmap;
 use risingwave_connector::sink::catalog::SinkId;
 use risingwave_connector::sink::SinkParam;
 use risingwave_pb::connector_service::coordinate_request::Msg;
@@ -357,7 +357,7 @@ mod tests {
     use futures::{FutureExt, StreamExt};
     use itertools::Itertools;
     use rand::seq::SliceRandom;
-    use risingwave_common::buffer::{Bitmap, BitmapBuilder};
+    use risingwave_common::bitmap::{Bitmap, BitmapBuilder};
     use risingwave_common::hash::VirtualNode;
     use risingwave_connector::sink::catalog::{SinkId, SinkType};
     use risingwave_connector::sink::{SinkCommitCoordinator, SinkError, SinkParam};
diff --git a/src/meta/src/manager/sink_coordination/mod.rs b/src/meta/src/manager/sink_coordination/mod.rs
index 68f3f44624e38..ab44965891d5f 100644
--- a/src/meta/src/manager/sink_coordination/mod.rs
+++ b/src/meta/src/manager/sink_coordination/mod.rs
@@ -17,7 +17,7 @@ mod manager;
 
 use futures::stream::BoxStream;
 pub use manager::SinkCoordinatorManager;
-use risingwave_common::buffer::Bitmap;
+use risingwave_common::bitmap::Bitmap;
 use risingwave_connector::sink::SinkParam;
 use risingwave_pb::connector_service::{CoordinateRequest, CoordinateResponse};
 use tokio::sync::mpsc::Sender;
diff --git a/src/meta/src/manager/streaming_job.rs b/src/meta/src/manager/streaming_job.rs
index 90d0781174b5d..0aca9eccf8974 100644
--- a/src/meta/src/manager/streaming_job.rs
+++ b/src/meta/src/manager/streaming_job.rs
@@ -19,7 +19,7 @@ use risingwave_common::current_cluster_version;
 use risingwave_common::util::epoch::Epoch;
 use risingwave_pb::catalog::{CreateType, Index, PbSource, Sink, Table};
 use risingwave_pb::ddl_service::TableJobType;
-use strum::EnumDiscriminants;
+use strum::{EnumDiscriminants, EnumIs};
 
 use super::{get_refed_secret_ids_from_sink, get_refed_secret_ids_from_source};
 use crate::model::FragmentId;
@@ -27,7 +27,7 @@ use crate::MetaResult;
 
 // This enum is used in order to re-use code in `DdlServiceImpl` for creating MaterializedView and
 // Sink.
-#[derive(Debug, Clone, EnumDiscriminants)]
+#[derive(Debug, Clone, EnumDiscriminants, EnumIs)]
 pub enum StreamingJob {
     MaterializedView(Table),
     Sink(Sink, Option<(Table, Option<PbSource>)>),
@@ -241,6 +241,16 @@ impl StreamingJob {
         }
     }
 
+    pub fn job_type_str(&self) -> &'static str {
+        match self {
+            StreamingJob::MaterializedView(_) => "materialized view",
+            StreamingJob::Sink(_, _) => "sink",
+            StreamingJob::Table(_, _, _) => "table",
+            StreamingJob::Index(_, _) => "index",
+            StreamingJob::Source(_) => "source",
+        }
+    }
+
     pub fn definition(&self) -> String {
         match self {
             Self::MaterializedView(table) => table.definition.clone(),
@@ -289,7 +299,8 @@ impl StreamingJob {
         }
     }
 
-    pub fn dependent_secret_refs(&self) -> MetaResult<HashSet<u32>> {
+    // Get the secret ids that are referenced by this job.
+    pub fn dependent_secret_ids(&self) -> MetaResult<HashSet<u32>> {
         match self {
             StreamingJob::Sink(sink, _) => Ok(get_refed_secret_ids_from_sink(sink)),
             StreamingJob::Table(source, _, _) => {
diff --git a/src/meta/src/manager/system_param/mod.rs b/src/meta/src/manager/system_param/mod.rs
index 0030d3c2d8f88..dbc4a743fa72c 100644
--- a/src/meta/src/manager/system_param/mod.rs
+++ b/src/meta/src/manager/system_param/mod.rs
@@ -21,7 +21,9 @@ use std::time::Duration;
 use anyhow::anyhow;
 use risingwave_common::system_param::common::CommonHandler;
 use risingwave_common::system_param::reader::SystemParamsReader;
-use risingwave_common::system_param::{check_missing_params, set_system_param};
+use risingwave_common::system_param::{
+    check_missing_params, set_system_param, TIME_TRAVEL_RETENTION_MS_KEY,
+};
 use risingwave_common::{for_all_params, key_of};
 use risingwave_pb::meta::subscribe_response::{Info, Operation};
 use risingwave_pb::meta::SystemParams;
@@ -32,6 +34,7 @@ use tracing::info;
 
 use self::model::SystemParamsModel;
 use super::NotificationManagerRef;
+use crate::hummock::time_travel::require_sql_meta_store_err;
 use crate::model::{InMemValTransaction, ValTransaction, VarTransaction};
 use crate::storage::{MetaStore, MetaStoreRef, Transaction};
 use crate::{MetaError, MetaResult};
@@ -66,6 +69,14 @@ impl SystemParamsManager {
             ));
         };
 
+        if params
+            .time_travel_retention_ms
+            .map(|r| r > 0)
+            .unwrap_or(false)
+        {
+            return Err(require_sql_meta_store_err().into());
+        }
+
         info!("system parameters: {:?}", params);
         check_missing_params(&params).map_err(|e| anyhow!(e))?;
 
@@ -86,6 +97,10 @@ impl SystemParamsManager {
     }
 
     pub async fn set_param(&self, name: &str, value: Option<String>) -> MetaResult<SystemParams> {
+        if name == TIME_TRAVEL_RETENTION_MS_KEY {
+            return Err(require_sql_meta_store_err().into());
+        }
+
         let mut params_guard = self.params.write().await;
         let params = params_guard.deref_mut();
         let mut mem_txn = VarTransaction::new(params);
diff --git a/src/meta/src/model/migration_plan.rs b/src/meta/src/model/migration_plan.rs
index b4703c9284fb4..338fa00911c9a 100644
--- a/src/meta/src/model/migration_plan.rs
+++ b/src/meta/src/model/migration_plan.rs
@@ -15,7 +15,7 @@
 use std::collections::HashMap;
 
 use prost::Message;
-use risingwave_pb::common::ParallelUnit;
+use risingwave_common::hash::WorkerSlotId;
 use risingwave_pb::meta::PbMigrationPlan;
 
 use crate::storage::{MetaStore, MetaStoreError, MetaStoreResult, Snapshot, DEFAULT_COLUMN_FAMILY};
@@ -25,11 +25,9 @@ pub const MIGRATION_PLAN_KEY: &[u8] = &[
     109, 105, 103, 114, 97, 116, 105, 111, 110, 95, 112, 108, 97, 110,
 ];
 
-type ParallelUnitId = u32;
-
 #[derive(Debug, Default, Clone)]
 pub struct MigrationPlan {
-    pub parallel_unit_plan: HashMap<ParallelUnitId, ParallelUnit>,
+    pub worker_slot_plan: HashMap<WorkerSlotId, WorkerSlotId>,
 }
 
 impl MigrationPlan {
@@ -75,7 +73,11 @@ impl MigrationPlan {
 impl From<PbMigrationPlan> for MigrationPlan {
     fn from(plan: PbMigrationPlan) -> Self {
         MigrationPlan {
-            parallel_unit_plan: plan.parallel_unit_migration_plan,
+            worker_slot_plan: plan
+                .worker_slot_migration_plan
+                .into_iter()
+                .map(|(k, v)| (WorkerSlotId::from(k), WorkerSlotId::from(v)))
+                .collect(),
         }
     }
 }
@@ -83,7 +85,11 @@ impl From<PbMigrationPlan> for MigrationPlan {
 impl From<MigrationPlan> for PbMigrationPlan {
     fn from(plan: MigrationPlan) -> Self {
         PbMigrationPlan {
-            parallel_unit_migration_plan: plan.parallel_unit_plan,
+            worker_slot_migration_plan: plan
+                .worker_slot_plan
+                .into_iter()
+                .map(|(k, v)| (k.into(), v.into()))
+                .collect(),
         }
     }
 }
diff --git a/src/meta/src/model/mod.rs b/src/meta/src/model/mod.rs
index 1ca22854c5def..07f57ff1d6f6d 100644
--- a/src/meta/src/model/mod.rs
+++ b/src/meta/src/model/mod.rs
@@ -207,7 +207,8 @@ macro_rules! for_all_metadata_models {
     ($macro:ident) => {
         $macro! {
             // These items should be included in a meta snapshot.
-            // So be sure to update meta backup/restore when adding new items.
+            // Make sure to update the meta backup&restore methods accordingly when adding new items,
+            // referring to https://github.com/risingwavelabs/risingwave/pull/15371/commits/c5a75320845a38cfb43241ddee16fd5c0e47833b
             { risingwave_pb::hummock::HummockVersionStats },
             { crate::hummock::model::CompactionGroup },
             { risingwave_pb::catalog::Database },
@@ -462,16 +463,20 @@ enum BTreeMapOp<V> {
 /// are stored in `staging`. On `commit`, it will apply the changes stored in `staging` to the in
 /// memory btree map. When serve `get` and `get_mut`, it merges the value stored in `staging` and
 /// `tree_ref`.
-pub struct BTreeMapTransaction<'a, K: Ord, V> {
+pub struct BTreeMapTransactionInner<K: Ord, V, P: DerefMut<Target = BTreeMap<K, V>>> {
     /// A reference to the original `BTreeMap`. All access to this field should be immutable,
     /// except when we commit the staging changes to the original map.
-    tree_ref: &'a mut BTreeMap<K, V>,
+    tree_ref: P,
     /// Store all the staging changes that will be applied to the original map on commit
     staging: BTreeMap<K, BTreeMapOp<V>>,
 }
 
-impl<'a, K: Ord + Debug, V: Clone> BTreeMapTransaction<'a, K, V> {
-    pub fn new(tree_ref: &'a mut BTreeMap<K, V>) -> BTreeMapTransaction<'a, K, V> {
+pub type BTreeMapTransaction<'a, K, V> = BTreeMapTransactionInner<K, V, &'a mut BTreeMap<K, V>>;
+
+impl<K: Ord + Debug, V: Clone, P: DerefMut<Target = BTreeMap<K, V>>>
+    BTreeMapTransactionInner<K, V, P>
+{
+    pub fn new(tree_ref: P) -> BTreeMapTransactionInner<K, V, P> {
         Self {
             tree_ref,
             staging: BTreeMap::default(),
@@ -481,7 +486,7 @@ impl<'a, K: Ord + Debug, V: Clone> BTreeMapTransaction<'a, K, V> {
     /// Start a `BTreeMapEntryTransaction` when the `key` exists
     #[allow(dead_code)]
     pub fn new_entry_txn(&mut self, key: K) -> Option<BTreeMapEntryTransaction<'_, K, V>> {
-        BTreeMapEntryTransaction::new(self.tree_ref, key, None)
+        BTreeMapEntryTransaction::new(&mut self.tree_ref, key, None)
     }
 
     /// Start a `BTreeMapEntryTransaction`. If the `key` does not exist, the the `default_val` will
@@ -492,17 +497,17 @@ impl<'a, K: Ord + Debug, V: Clone> BTreeMapTransaction<'a, K, V> {
         key: K,
         default_val: V,
     ) -> BTreeMapEntryTransaction<'_, K, V> {
-        BTreeMapEntryTransaction::new(self.tree_ref, key, Some(default_val))
+        BTreeMapEntryTransaction::new(&mut self.tree_ref, key, Some(default_val))
             .expect("default value is provided and should return `Some`")
     }
 
     /// Start a `BTreeMapEntryTransaction` that inserts the `val` into `key`.
     pub fn new_entry_insert_txn(&mut self, key: K, val: V) -> BTreeMapEntryTransaction<'_, K, V> {
-        BTreeMapEntryTransaction::new_insert(self.tree_ref, key, val)
+        BTreeMapEntryTransaction::new_insert(&mut self.tree_ref, key, val)
     }
 
     pub fn tree_ref(&self) -> &BTreeMap<K, V> {
-        self.tree_ref
+        &self.tree_ref
     }
 
     /// Get the value of the provided key by merging the staging value and the original value
@@ -578,7 +583,7 @@ impl<'a, K: Ord + Debug, V: Clone> BTreeMapTransaction<'a, K, V> {
         }
     }
 
-    pub fn commit_memory(self) {
+    pub fn commit_memory(mut self) {
         // Apply each op stored in the staging to original tree.
         for (k, op) in self.staging {
             match op {
@@ -593,14 +598,16 @@ impl<'a, K: Ord + Debug, V: Clone> BTreeMapTransaction<'a, K, V> {
     }
 }
 
-impl<'a, K: Ord + Debug, V: Clone> InMemValTransaction for BTreeMapTransaction<'a, K, V> {
+impl<K: Ord + Debug, V: Clone, P: DerefMut<Target = BTreeMap<K, V>>> InMemValTransaction
+    for BTreeMapTransactionInner<K, V, P>
+{
     fn commit(self) {
         self.commit_memory();
     }
 }
 
-impl<'a, K: Ord + Debug, V: Transactional<TXN> + Clone, TXN> ValTransaction<TXN>
-    for BTreeMapTransaction<'a, K, V>
+impl<K: Ord + Debug, V: Transactional<TXN> + Clone, P: DerefMut<Target = BTreeMap<K, V>>, TXN>
+    ValTransaction<TXN> for BTreeMapTransactionInner<K, V, P>
 {
     async fn apply_to_txn(&self, txn: &mut TXN) -> MetadataModelResult<()> {
         // Add the staging operation to txn
@@ -696,6 +703,76 @@ impl<'a, K: Ord, V: PartialEq + Transactional<TXN>, TXN> ValTransaction<TXN>
     }
 }
 
+impl<T: InMemValTransaction> InMemValTransaction for Option<T> {
+    fn commit(self) {
+        if let Some(inner) = self {
+            inner.commit();
+        }
+    }
+}
+
+impl<T: ValTransaction<TXN>, TXN> ValTransaction<TXN> for Option<T> {
+    async fn apply_to_txn(&self, txn: &mut TXN) -> MetadataModelResult<()> {
+        if let Some(inner) = &self {
+            inner.apply_to_txn(txn).await?;
+        }
+        Ok(())
+    }
+}
+
+pub struct DerefMutForward<
+    Inner,
+    Target,
+    P: DerefMut<Target = Inner>,
+    F: Fn(&Inner) -> &Target,
+    FMut: Fn(&mut Inner) -> &mut Target,
+> {
+    ptr: P,
+    f: F,
+    f_mut: FMut,
+}
+
+impl<
+        Inner,
+        Target,
+        P: DerefMut<Target = Inner>,
+        F: Fn(&Inner) -> &Target,
+        FMut: Fn(&mut Inner) -> &mut Target,
+    > DerefMutForward<Inner, Target, P, F, FMut>
+{
+    pub fn new(ptr: P, f: F, f_mut: FMut) -> Self {
+        Self { ptr, f, f_mut }
+    }
+}
+
+impl<
+        Inner,
+        Target,
+        P: DerefMut<Target = Inner>,
+        F: Fn(&Inner) -> &Target,
+        FMut: Fn(&mut Inner) -> &mut Target,
+    > Deref for DerefMutForward<Inner, Target, P, F, FMut>
+{
+    type Target = Target;
+
+    fn deref(&self) -> &Self::Target {
+        (self.f)(&self.ptr)
+    }
+}
+
+impl<
+        Inner,
+        Target,
+        P: DerefMut<Target = Inner>,
+        F: Fn(&Inner) -> &Target,
+        FMut: Fn(&mut Inner) -> &mut Target,
+    > DerefMut for DerefMutForward<Inner, Target, P, F, FMut>
+{
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        (self.f_mut)(&mut self.ptr)
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use itertools::Itertools;
diff --git a/src/meta/src/model/stream.rs b/src/meta/src/model/stream.rs
index 144a33ae7b24b..68e45863cfed6 100644
--- a/src/meta/src/model/stream.rs
+++ b/src/meta/src/model/stream.rs
@@ -17,9 +17,9 @@ use std::ops::AddAssign;
 
 use itertools::Itertools;
 use risingwave_common::catalog::TableId;
-use risingwave_common::hash::ParallelUnitId;
+use risingwave_common::hash::WorkerSlotId;
 use risingwave_connector::source::SplitImpl;
-use risingwave_pb::common::{ParallelUnit, ParallelUnitMapping};
+use risingwave_pb::common::PbActorLocation;
 use risingwave_pb::meta::table_fragments::actor_status::ActorState;
 use risingwave_pb::meta::table_fragments::{ActorStatus, Fragment, State};
 use risingwave_pb::meta::table_parallelism::{
@@ -44,7 +44,7 @@ const TABLE_FRAGMENTS_CF_NAME: &str = "cf/table_fragments";
 /// The parallelism for a `TableFragments`.
 #[derive(Debug, Copy, Clone, Eq, PartialEq)]
 pub enum TableParallelism {
-    /// This is when the system decides the parallelism, based on the available parallel units.
+    /// This is when the system decides the parallelism, based on the available worker parallelisms.
     Adaptive,
     /// We set this when the `TableFragments` parallelism is changed.
     /// All fragments which are part of the `TableFragment` will have the same parallelism as this.
@@ -164,6 +164,8 @@ impl MetadataModel for TableFragments {
             actor_splits: build_actor_connector_splits(&self.actor_splits),
             ctx: Some(self.ctx.to_protobuf()),
             parallelism: Some(self.assigned_parallelism.into()),
+            node_label: "".to_string(),
+            backfill_done: true,
         }
     }
 
@@ -174,9 +176,11 @@ impl MetadataModel for TableFragments {
             parallelism: Some(Parallelism::Custom(PbCustomParallelism {})),
         };
 
+        let state = prost.state();
+
         Self {
             table_id: TableId::new(prost.table_id),
-            state: prost.state(),
+            state,
             fragments: prost.fragments.into_iter().collect(),
             actor_status: prost.actor_status.into_iter().collect(),
             actor_splits: build_actor_split_impls(&prost.actor_splits),
@@ -203,21 +207,21 @@ impl TableFragments {
     }
 
     /// Create a new `TableFragments` with state of `Initial`, with the status of actors set to
-    /// `Inactive` on the given parallel units.
+    /// `Inactive` on the given workers.
     pub fn new(
         table_id: TableId,
         fragments: BTreeMap<FragmentId, Fragment>,
-        actor_locations: &BTreeMap<ActorId, ParallelUnit>,
+        actor_locations: &BTreeMap<ActorId, WorkerSlotId>,
         ctx: StreamContext,
         table_parallelism: TableParallelism,
     ) -> Self {
         let actor_status = actor_locations
             .iter()
-            .map(|(&actor_id, parallel_unit)| {
+            .map(|(&actor_id, worker_slot_id)| {
                 (
                     actor_id,
                     ActorStatus {
-                        parallel_unit: Some(parallel_unit.clone()),
+                        location: PbActorLocation::from_worker(worker_slot_id.worker_id()),
                         state: ActorState::Inactive as i32,
                     },
                 )
@@ -279,24 +283,6 @@ impl TableFragments {
         self.state = state;
     }
 
-    /// Returns mview fragment vnode mapping.
-    /// Note that: the sink fragment is also stored as `TableFragments`, it's possible that
-    /// there's no fragment with `FragmentTypeFlag::Mview` exists.
-    pub fn mview_vnode_mapping(&self) -> Option<(FragmentId, ParallelUnitMapping)> {
-        self.fragments
-            .values()
-            .find(|fragment| {
-                (fragment.get_fragment_type_mask() & FragmentTypeFlag::Mview as u32) != 0
-            })
-            .map(|fragment| {
-                (
-                    fragment.fragment_id,
-                    // vnode mapping is always `Some`, even for singletons.
-                    fragment.vnode_mapping.clone().unwrap(),
-                )
-            })
-    }
-
     /// Update state of all actors
     pub fn update_actors_state(&mut self, state: ActorState) {
         for actor_status in self.actor_status.values_mut() {
@@ -493,7 +479,7 @@ impl TableFragments {
     pub fn worker_actor_states(&self) -> BTreeMap<WorkerId, Vec<(ActorId, ActorState)>> {
         let mut map = BTreeMap::default();
         for (&actor_id, actor_status) in &self.actor_status {
-            let node_id = actor_status.get_parallel_unit().unwrap().worker_node_id as WorkerId;
+            let node_id = actor_status.worker_id() as WorkerId;
             map.entry(node_id)
                 .or_insert_with(Vec::new)
                 .push((actor_id, actor_status.state()));
@@ -505,43 +491,18 @@ impl TableFragments {
     pub fn worker_actor_ids(&self) -> BTreeMap<WorkerId, Vec<ActorId>> {
         let mut map = BTreeMap::default();
         for (&actor_id, actor_status) in &self.actor_status {
-            let node_id = actor_status.get_parallel_unit().unwrap().worker_node_id as WorkerId;
+            let node_id = actor_status.worker_id() as WorkerId;
             map.entry(node_id).or_insert_with(Vec::new).push(actor_id);
         }
         map
     }
 
-    pub fn worker_parallel_units(&self) -> HashMap<WorkerId, HashSet<ParallelUnitId>> {
-        let mut map = HashMap::new();
-        for actor_status in self.actor_status.values() {
-            map.entry(actor_status.get_parallel_unit().unwrap().worker_node_id)
-                .or_insert_with(HashSet::new)
-                .insert(actor_status.get_parallel_unit().unwrap().id);
-        }
-        map
-    }
-
-    pub fn update_vnode_mapping(&mut self, migrate_map: &HashMap<ParallelUnitId, ParallelUnit>) {
-        for fragment in self.fragments.values_mut() {
-            if let Some(mapping) = &mut fragment.vnode_mapping {
-                mapping.data.iter_mut().for_each(|id| {
-                    if migrate_map.contains_key(id) {
-                        *id = migrate_map.get(id).unwrap().id;
-                    }
-                });
-            }
-        }
-    }
-
     /// Returns the status of actors group by worker id.
     pub fn worker_actors(&self, include_inactive: bool) -> BTreeMap<WorkerId, Vec<StreamActor>> {
         let mut actors = BTreeMap::default();
         for fragment in self.fragments.values() {
             for actor in &fragment.actors {
-                let node_id = self.actor_status[&actor.actor_id]
-                    .get_parallel_unit()
-                    .unwrap()
-                    .worker_node_id as WorkerId;
+                let node_id = self.actor_status[&actor.actor_id].worker_id() as WorkerId;
                 if !include_inactive
                     && self.actor_status[&actor.actor_id].state == ActorState::Inactive as i32
                 {
diff --git a/src/meta/src/rpc/ddl_controller.rs b/src/meta/src/rpc/ddl_controller.rs
index d88e517a5e4e4..c5a750ee7230a 100644
--- a/src/meta/src/rpc/ddl_controller.rs
+++ b/src/meta/src/rpc/ddl_controller.rs
@@ -18,28 +18,27 @@ use std::num::NonZeroUsize;
 use std::sync::Arc;
 use std::time::Duration;
 
-use aes_siv::aead::generic_array::GenericArray;
-use aes_siv::aead::Aead;
-use aes_siv::{Aes128SivAead, KeyInit};
-use anyhow::Context;
+use anyhow::{anyhow, Context};
 use itertools::Itertools;
-use rand::{Rng, RngCore};
+use rand::Rng;
+use risingwave_common::bitmap::Bitmap;
 use risingwave_common::config::DefaultParallelism;
-use risingwave_common::hash::{ParallelUnitMapping, VirtualNode};
+use risingwave_common::hash::{ActorMapping, VirtualNode};
+use risingwave_common::secret::SecretEncryption;
 use risingwave_common::system_param::reader::SystemParamsRead;
 use risingwave_common::util::column_index_mapping::ColIndexMapping;
 use risingwave_common::util::epoch::Epoch;
 use risingwave_common::util::stream_graph_visitor::{
     visit_fragment, visit_stream_node, visit_stream_node_cont_mut,
 };
-use risingwave_common::{bail, current_cluster_version};
-use risingwave_connector::dispatch_source_prop;
+use risingwave_common::{bail, current_cluster_version, hash, must_match};
 use risingwave_connector::error::ConnectorError;
 use risingwave_connector::source::cdc::CdcSourceType;
 use risingwave_connector::source::{
     ConnectorProperties, SourceEnumeratorContext, SourceProperties, SplitEnumerator,
     UPSTREAM_SOURCE_KEY,
 };
+use risingwave_connector::{dispatch_source_prop, WithOptionsSecResolved};
 use risingwave_meta_model_v2::object::ObjectType;
 use risingwave_meta_model_v2::ObjectId;
 use risingwave_pb::catalog::connection::private_link_service::PbPrivateLinkProvider;
@@ -47,13 +46,14 @@ use risingwave_pb::catalog::connection::PrivateLinkService;
 use risingwave_pb::catalog::source::OptionalAssociatedTableId;
 use risingwave_pb::catalog::table::OptionalAssociatedSourceId;
 use risingwave_pb::catalog::{
-    connection, Comment, Connection, CreateType, Database, Function, PbSource, PbTable, Schema,
-    Secret, Sink, Source, Subscription, Table, View,
+    connection, Comment, Connection, CreateType, Database, Function, PbSink, PbSource, PbTable,
+    Schema, Secret, Sink, Source, Subscription, Table, View,
 };
 use risingwave_pb::ddl_service::alter_owner_request::Object;
 use risingwave_pb::ddl_service::{
     alter_name_request, alter_set_schema_request, DdlProgress, TableJobType,
 };
+use risingwave_pb::meta::table_fragments::fragment::FragmentDistributionType;
 use risingwave_pb::meta::table_fragments::PbFragment;
 use risingwave_pb::meta::PbTableParallelism;
 use risingwave_pb::stream_plan::stream_node::NodeBody;
@@ -61,7 +61,6 @@ use risingwave_pb::stream_plan::{
     Dispatcher, DispatcherType, FragmentTypeFlag, MergeNode, PbStreamFragmentGraph,
     StreamFragmentGraph as StreamFragmentGraphProto,
 };
-use serde::{Deserialize, Serialize};
 use thiserror_ext::AsReport;
 use tokio::sync::Semaphore;
 use tokio::time::sleep;
@@ -69,11 +68,10 @@ use tracing::log::warn;
 use tracing::Instrument;
 
 use crate::barrier::BarrierManagerRef;
-use crate::error::MetaErrorInner;
 use crate::manager::{
-    CatalogManagerRef, ConnectionId, DatabaseId, FragmentManagerRef, FunctionId, IdCategory,
-    IdCategoryType, IndexId, LocalNotification, MetaSrvEnv, MetadataManager, MetadataManagerV1,
-    NotificationVersion, RelationIdEnum, SchemaId, SecretId, SinkId, SourceId,
+    CatalogManagerRef, ConnectionId, DatabaseId, DdlType, FragmentManagerRef, FunctionId,
+    IdCategory, IdCategoryType, IndexId, LocalNotification, MetaSrvEnv, MetadataManager,
+    MetadataManagerV1, NotificationVersion, RelationIdEnum, SchemaId, SecretId, SinkId, SourceId,
     StreamingClusterInfo, StreamingJob, StreamingJobDiscriminants, SubscriptionId, TableId, UserId,
     ViewId, IGNORED_NOTIFICATION_VERSION,
 };
@@ -160,12 +158,6 @@ pub enum DdlCommand {
     DropSubscription(SubscriptionId, DropMode),
 }
 
-#[derive(Deserialize, Serialize)]
-struct SecretEncryption {
-    nonce: [u8; 16],
-    ciphertext: Vec<u8>,
-}
-
 impl DdlCommand {
     fn allow_in_recovery(&self) -> bool {
         match self {
@@ -362,7 +354,7 @@ impl DdlController {
         tokio::spawn(fut).await.unwrap()
     }
 
-    pub async fn get_ddl_progress(&self) -> Vec<DdlProgress> {
+    pub async fn get_ddl_progress(&self) -> MetaResult<Vec<DdlProgress>> {
         self.barrier_manager.get_ddl_progress().await
     }
 
@@ -396,7 +388,7 @@ impl DdlController {
                 .await?
                 .is_empty()
         {
-            bail!("There are background creating jobs, please try again later")
+            bail!("The system is creating jobs in the background, please try again later")
         }
 
         self.stream_manager
@@ -629,44 +621,38 @@ impl DdlController {
     async fn create_secret(&self, mut secret: Secret) -> MetaResult<NotificationVersion> {
         // The 'secret' part of the request we receive from the frontend is in plaintext;
         // here, we need to encrypt it before storing it in the catalog.
+        let secret_plain_payload = secret.value.clone();
+        let secret_store_private_key = self
+            .env
+            .opts
+            .secret_store_private_key
+            .clone()
+            .ok_or_else(|| anyhow!("secret_store_private_key is not configured"))?;
 
         let encrypted_payload = {
-            let data = secret.get_value().as_slice();
-            let key = self.env.opts.secret_store_private_key.as_slice();
-            let encrypt_key = {
-                let mut k = key[..(std::cmp::min(key.len(), 32))].to_vec();
-                k.resize_with(32, || 0);
-                k
-            };
-
-            let mut rng = rand::thread_rng();
-            let mut nonce: [u8; 16] = [0; 16];
-            rng.fill_bytes(&mut nonce);
-            let nonce_array = GenericArray::from_slice(&nonce);
-            let cipher = Aes128SivAead::new(encrypt_key.as_slice().into());
-
-            let ciphertext = cipher.encrypt(nonce_array, data).map_err(|e| {
-                MetaError::from(MetaErrorInner::InvalidParameter(format!(
-                    "failed to encrypt secret {}: {:?}",
-                    secret.name, e
-                )))
-            })?;
-            bincode::serialize(&SecretEncryption { nonce, ciphertext }).map_err(|e| {
-                MetaError::from(MetaErrorInner::InvalidParameter(format!(
-                    "failed to serialize secret {}: {:?}",
-                    secret.name,
-                    e.as_report()
-                )))
-            })?
+            let encrypted_secret = SecretEncryption::encrypt(
+                secret_store_private_key.as_slice(),
+                secret.get_value().as_slice(),
+            )
+            .context(format!("failed to encrypt secret {}", secret.name))?;
+            encrypted_secret
+                .serialize()
+                .context(format!("failed to serialize secret {}", secret.name))?
         };
         secret.value = encrypted_payload;
 
         match &self.metadata_manager {
             MetadataManager::V1(mgr) => {
                 secret.id = self.gen_unique_id::<{ IdCategory::Secret }>().await?;
-                mgr.catalog_manager.create_secret(secret).await
+                mgr.catalog_manager
+                    .create_secret(secret, secret_plain_payload)
+                    .await
+            }
+            MetadataManager::V2(mgr) => {
+                mgr.catalog_controller
+                    .create_secret(secret, secret_plain_payload)
+                    .await
             }
-            MetadataManager::V2(mgr) => mgr.catalog_controller.create_secret(secret).await,
         }
     }
 
@@ -921,20 +907,22 @@ impl DdlController {
             None => None,
         };
 
+        let stream_job_clone_for_err_handle = stream_job.clone();
+
         // 4. Build and persist stream job.
         let result: MetaResult<_> = try {
             tracing::debug!(id = stream_job.id(), "building stream job");
             let (ctx, table_fragments) = self
                 .build_stream_job(
                     stream_ctx,
-                    &stream_job,
+                    stream_job,
                     fragment_graph,
                     affected_table_replace_info,
                 )
                 .await?;
 
             // Do some type-specific work for each type of stream job.
-            match stream_job {
+            match &ctx.streaming_job {
                 StreamingJob::Table(None, ref table, TableJobType::SharedCdcSource) => {
                     Self::validate_cdc_table(table, &table_fragments)
                         .await
@@ -944,16 +932,7 @@ impl DdlController {
                     // Register the source on the connector node.
                     self.source_manager.register_source(source).await?;
                 }
-                StreamingJob::Sink(ref sink, ref mut target_table) => {
-                    // When sinking into table occurs, some variables of the target table may be modified,
-                    // such as `fragment_id` being altered by `prepare_replace_table`.
-                    // At this point, it’s necessary to update the table info carried with the sink.
-                    if let Some((StreamingJob::Table(source, table, _), ..)) =
-                        &ctx.replace_table_job_info
-                    {
-                        *target_table = Some((table.clone(), source.clone()));
-                    }
-
+                StreamingJob::Sink(ref sink, _) => {
                     // Validate the sink on the connector node.
                     validate_sink(sink).await?;
                 }
@@ -970,6 +949,7 @@ impl DdlController {
         let (ctx, table_fragments) = match result {
             Ok(r) => r,
             Err(e) => {
+                let stream_job = stream_job_clone_for_err_handle;
                 tracing::error!(error = %e.as_report(), id = stream_job.id(), "failed to create streaming job");
                 self.cancel_stream_job(&stream_job, internal_tables, Some(&e))
                     .await?;
@@ -977,14 +957,13 @@ impl DdlController {
             }
         };
 
-        match (create_type, &stream_job) {
+        match (create_type, &ctx.streaming_job) {
             (CreateType::Foreground, _)
             | (CreateType::Unspecified, _)
             // FIXME(kwannoel): Unify background stream's creation path with MV below.
             | (CreateType::Background, &StreamingJob::Sink(_, _)) => {
                 self.create_streaming_job_inner(
                     mgr,
-                    stream_job,
                     table_fragments,
                     ctx,
                     internal_tables,
@@ -994,12 +973,11 @@ impl DdlController {
             (CreateType::Background, &StreamingJob::MaterializedView(_)) => {
                 let ctrl = self.clone();
                 let mgr = mgr.clone();
-                let stream_job_id = stream_job.id();
+                let stream_job_id = ctx.streaming_job.id();
                 let fut = async move {
                     let result = ctrl
                         .create_streaming_job_inner(
                             &mgr,
-                            stream_job,
                             table_fragments,
                             ctx,
                             internal_tables,
@@ -1018,7 +996,7 @@ impl DdlController {
                 Ok(IGNORED_NOTIFICATION_VERSION)
             }
             (CreateType::Background, _) => {
-                let d: StreamingJobDiscriminants = stream_job.into();
+                let d: StreamingJobDiscriminants = ctx.streaming_job.into();
                 bail!("background_ddl not supported for: {:?}", d)
             }
         }
@@ -1053,8 +1031,11 @@ impl DdlController {
                 actor.nodes.as_ref().unwrap().node_body
                 && let Some(ref cdc_table_desc) = stream_cdc_scan.cdc_table_desc
             {
-                let properties = cdc_table_desc.connect_properties.clone();
-                let mut props = ConnectorProperties::extract(properties, true)?;
+                let options_with_secret = WithOptionsSecResolved::new(
+                    cdc_table_desc.connect_properties.clone(),
+                    cdc_table_desc.secret_refs.clone(),
+                );
+                let mut props = ConnectorProperties::extract(options_with_secret, true)?;
                 props.init_from_pb_cdc_table_desc(cdc_table_desc);
 
                 dispatch_source_prop!(props, props, {
@@ -1103,53 +1084,61 @@ impl DdlController {
             }
         }
 
-        let table = streaming_job.table().unwrap();
+        let target_table = streaming_job.table().unwrap();
 
         let target_fragment_id =
             union_fragment_id.expect("fragment of placeholder merger not found");
 
         if let Some(creating_sink_table_fragments) = creating_sink_table_fragments {
             let sink_fragment = creating_sink_table_fragments.sink_fragment().unwrap();
-
+            let sink = sink.expect("sink not found");
             Self::inject_replace_table_plan_for_sink(
-                sink.map(|sink| sink.id),
+                Some(sink.id),
                 &sink_fragment,
-                table,
+                target_table,
                 &mut replace_table_ctx,
                 &mut table_fragments,
                 target_fragment_id,
+                None,
             );
         }
 
         let [table_catalog]: [_; 1] = mgr
-            .get_table_catalog_by_ids(vec![table.id])
+            .get_table_catalog_by_ids(vec![target_table.id])
             .await?
             .try_into()
             .expect("Target table should exist in sink into table");
 
-        assert_eq!(table_catalog.incoming_sinks, table.incoming_sinks);
+        assert_eq!(table_catalog.incoming_sinks, target_table.incoming_sinks);
 
         {
-            for sink_id in &table_catalog.incoming_sinks {
+            let catalogs = mgr
+                .get_sink_catalog_by_ids(&table_catalog.incoming_sinks)
+                .await?;
+
+            for sink in catalogs {
+                let sink_id = sink.id;
+
                 if let Some(dropping_sink_id) = dropping_sink_id
-                    && *sink_id == dropping_sink_id
+                    && sink_id == dropping_sink_id
                 {
                     continue;
                 };
 
                 let sink_table_fragments = mgr
-                    .get_job_fragments_by_id(&risingwave_common::catalog::TableId::new(*sink_id))
+                    .get_job_fragments_by_id(&risingwave_common::catalog::TableId::new(sink_id))
                     .await?;
 
                 let sink_fragment = sink_table_fragments.sink_fragment().unwrap();
 
                 Self::inject_replace_table_plan_for_sink(
-                    Some(*sink_id),
+                    Some(sink_id),
                     &sink_fragment,
-                    table,
+                    target_table,
                     &mut replace_table_ctx,
                     &mut table_fragments,
                     target_fragment_id,
+                    Some(&sink.unique_identity()),
                 );
             }
         }
@@ -1170,13 +1159,14 @@ impl DdlController {
         Ok((replace_table_ctx, table_fragments))
     }
 
-    fn inject_replace_table_plan_for_sink(
+    pub(crate) fn inject_replace_table_plan_for_sink(
         sink_id: Option<u32>,
         sink_fragment: &PbFragment,
         table: &Table,
         replace_table_ctx: &mut ReplaceTableContext,
         table_fragments: &mut TableFragments,
         target_fragment_id: FragmentId,
+        unique_identity: Option<&str>,
     ) {
         let sink_actor_ids = sink_fragment
             .actors
@@ -1195,8 +1185,18 @@ impl DdlController {
             .map(|actor| actor.actor_id)
             .collect_vec();
 
-        let output_indices = table
-            .columns
+        let mut sink_fields = None;
+
+        for actor in &sink_fragment.actors {
+            if let Some(node) = &actor.nodes {
+                sink_fields = Some(node.fields.clone());
+                break;
+            }
+        }
+
+        let sink_fields = sink_fields.expect("sink fields not found");
+
+        let output_indices = sink_fields
             .iter()
             .enumerate()
             .map(|(idx, _)| idx as _)
@@ -1204,19 +1204,25 @@ impl DdlController {
 
         let dist_key_indices = table.distribution_key.iter().map(|i| *i as _).collect_vec();
 
-        let mapping = downstream_actor_ids
-            .iter()
-            .map(|id| {
-                let actor_status = table_fragments.actor_status.get(id).unwrap();
-                let parallel_unit_id = actor_status.parallel_unit.as_ref().unwrap().id;
-
-                (parallel_unit_id, *id)
-            })
-            .collect();
+        let mapping = match union_fragment.get_distribution_type().unwrap() {
+            FragmentDistributionType::Unspecified => unreachable!(),
+            FragmentDistributionType::Single => None,
+            FragmentDistributionType::Hash => {
+                let actor_bitmaps: HashMap<_, _> = union_fragment
+                    .actors
+                    .iter()
+                    .map(|actor| {
+                        (
+                            actor.actor_id as hash::ActorId,
+                            Bitmap::from(actor.vnode_bitmap.as_ref().unwrap()),
+                        )
+                    })
+                    .collect();
 
-        let actor_mapping =
-            ParallelUnitMapping::from_protobuf(union_fragment.vnode_mapping.as_ref().unwrap())
-                .to_actor(&mapping);
+                let actor_mapping = ActorMapping::from_bitmaps(&actor_bitmaps);
+                Some(actor_mapping)
+            }
+        };
 
         let upstream_actors = sink_fragment.get_actors();
 
@@ -1227,7 +1233,7 @@ impl DdlController {
                     r#type: DispatcherType::Hash as _,
                     dist_key_indices: dist_key_indices.clone(),
                     output_indices: output_indices.clone(),
-                    hash_mapping: Some(actor_mapping.to_protobuf()),
+                    hash_mapping: mapping.as_ref().map(|m| m.to_protobuf()),
                     dispatcher_id: union_fragment.fragment_id as _,
                     downstream_actor_id: downstream_actor_ids.clone(),
                 }],
@@ -1235,29 +1241,47 @@ impl DdlController {
         }
 
         let upstream_fragment_id = sink_fragment.fragment_id;
+
         for actor in &mut union_fragment.actors {
             if let Some(node) = &mut actor.nodes {
-                let fields = node.fields.clone();
-
                 visit_stream_node_cont_mut(node, |node| {
                     if let Some(NodeBody::Union(_)) = &mut node.node_body {
-                        for input in &mut node.input {
-                            if let Some(NodeBody::Merge(merge_node)) = &mut input.node_body
-                                && merge_node.upstream_actor_id.is_empty()
-                            {
-                                if let Some(sink_id) = sink_id {
-                                    input.identity =
-                                        format!("MergeExecutor(from sink {})", sink_id);
+                        for input_project_node in &mut node.input {
+                            if let Some(NodeBody::Project(_)) = &mut input_project_node.node_body {
+                                let merge_stream_node =
+                                    input_project_node.input.iter_mut().exactly_one().unwrap();
+
+                                // we need to align nodes here
+                                if input_project_node.identity.as_str()
+                                    != unique_identity
+                                        .unwrap_or(PbSink::UNIQUE_IDENTITY_FOR_CREATING_TABLE_SINK)
+                                {
+                                    continue;
                                 }
 
-                                *merge_node = MergeNode {
-                                    upstream_actor_id: sink_actor_ids.clone(),
-                                    upstream_fragment_id,
-                                    upstream_dispatcher_type: DispatcherType::Hash as _,
-                                    fields: fields.clone(),
-                                };
+                                if let Some(NodeBody::Merge(merge_node)) =
+                                    &mut merge_stream_node.node_body
+                                    && merge_node.upstream_actor_id.is_empty()
+                                {
+                                    if let Some(sink_id) = sink_id {
+                                        merge_stream_node.identity =
+                                            format!("MergeExecutor(from sink {})", sink_id);
+
+                                        input_project_node.identity =
+                                            format!("ProjectExecutor(from sink {})", sink_id);
+                                    }
+
+                                    *merge_node = MergeNode {
+                                        upstream_actor_id: sink_actor_ids.clone(),
+                                        upstream_fragment_id,
+                                        upstream_dispatcher_type: DispatcherType::Hash as _,
+                                        fields: sink_fields.to_vec(),
+                                    };
 
-                                return false;
+                                    merge_stream_node.fields = sink_fields.to_vec();
+
+                                    return false;
+                                }
                             }
                         }
                     }
@@ -1280,15 +1304,15 @@ impl DdlController {
     async fn create_streaming_job_inner(
         &self,
         mgr: &MetadataManagerV1,
-        stream_job: StreamingJob,
         table_fragments: TableFragments,
         ctx: CreateStreamingJobContext,
         internal_tables: Vec<Table>,
     ) -> MetaResult<NotificationVersion> {
+        let stream_job = ctx.streaming_job.clone();
         let job_id = stream_job.id();
         tracing::debug!(id = job_id, "creating stream job");
 
-        let result: MetaResult<()> = try {
+        let result: MetaResult<NotificationVersion> = try {
             // Add table fragments to meta store with state: `State::Initial`.
             mgr.fragment_manager
                 .start_create_table_fragments(table_fragments.clone())
@@ -1299,44 +1323,41 @@ impl DdlController {
                 .await?
         };
 
-        if let Err(e) = result {
-            match stream_job.create_type() {
-                CreateType::Background => {
-                    tracing::error!(id = job_id, error = %e.as_report(), "finish stream job failed");
-                    let should_cancel = match mgr
-                        .fragment_manager
-                        .select_table_fragments_by_table_id(&job_id.into())
-                        .await
-                    {
-                        Err(err) => err.is_fragment_not_found(),
-                        Ok(table_fragments) => table_fragments.is_initial(),
-                    };
-                    if should_cancel {
-                        // If the table fragments are not found or in initial state, it means that the stream job has not been created.
-                        // We need to cancel the stream job.
+        match result {
+            Err(e) => {
+                match stream_job.create_type() {
+                    CreateType::Background => {
+                        tracing::error!(id = job_id, error = %e.as_report(), "finish stream job failed");
+                        let should_cancel = match mgr
+                            .fragment_manager
+                            .select_table_fragments_by_table_id(&job_id.into())
+                            .await
+                        {
+                            Err(err) => err.is_fragment_not_found(),
+                            Ok(table_fragments) => table_fragments.is_initial(),
+                        };
+                        if should_cancel {
+                            // If the table fragments are not found or in initial state, it means that the stream job has not been created.
+                            // We need to cancel the stream job.
+                            self.cancel_stream_job(&stream_job, internal_tables, Some(&e))
+                                .await?;
+                        } else {
+                            // NOTE: This assumes that we will trigger recovery,
+                            // and recover stream job progress.
+                        }
+                    }
+                    _ => {
                         self.cancel_stream_job(&stream_job, internal_tables, Some(&e))
                             .await?;
-                    } else {
-                        // NOTE: This assumes that we will trigger recovery,
-                        // and recover stream job progress.
                     }
                 }
-                _ => {
-                    self.cancel_stream_job(&stream_job, internal_tables, Some(&e))
-                        .await?;
-                }
+                Err(e)
             }
-            return Err(e);
-        };
-
-        tracing::debug!(id = job_id, "finishing stream job");
-        let version = mgr
-            .catalog_manager
-            .finish_stream_job(stream_job, internal_tables)
-            .await?;
-        tracing::debug!(id = job_id, "finished stream job");
-
-        Ok(version)
+            Ok(version) => {
+                tracing::info!(id = job_id, "finish stream job succeeded");
+                Ok(version)
+            }
+        }
     }
 
     async fn drop_streaming_job(
@@ -1478,6 +1499,7 @@ impl DdlController {
                             None,
                             None,
                             Some(sink_id),
+                            vec![],
                         )
                         .await?;
                 }
@@ -1503,31 +1525,29 @@ impl DdlController {
     ) -> MetaResult<NonZeroUsize> {
         const MAX_PARALLELISM: NonZeroUsize = NonZeroUsize::new(VirtualNode::COUNT).unwrap();
 
-        if cluster_info.parallel_units.is_empty() {
-            return Err(MetaError::unavailable(
-                "No available parallel units to schedule",
-            ));
+        let available_parallelism = cluster_info.parallelism();
+        if available_parallelism == 0 {
+            return Err(MetaError::unavailable("No available slots to schedule"));
         }
 
-        let available_parallel_units =
-            NonZeroUsize::new(cluster_info.parallel_units.len()).unwrap();
+        let available_parallelism = NonZeroUsize::new(available_parallelism).unwrap();
 
-        // Use configured parallel units if no default parallelism is specified.
+        // Use configured parallelism if no default parallelism is specified.
         let parallelism =
             specified_parallelism.unwrap_or_else(|| match &self.env.opts.default_parallelism {
-                DefaultParallelism::Full => available_parallel_units,
+                DefaultParallelism::Full => available_parallelism,
                 DefaultParallelism::Default(num) => *num,
             });
 
-        if parallelism > available_parallel_units {
+        if parallelism > available_parallelism {
             return Err(MetaError::unavailable(format!(
-                "Not enough parallel units to schedule, required: {}, available: {}",
-                parallelism, available_parallel_units
+                "Not enough parallelism to schedule, required: {}, available: {}",
+                parallelism, available_parallelism
             )));
         }
 
-        if available_parallel_units > MAX_PARALLELISM {
-            tracing::warn!("Too many parallel units, use {} instead", MAX_PARALLELISM);
+        if available_parallelism > MAX_PARALLELISM {
+            tracing::warn!("Too many parallelism, use {} instead", MAX_PARALLELISM);
             Ok(MAX_PARALLELISM)
         } else {
             Ok(parallelism)
@@ -1541,7 +1561,7 @@ impl DdlController {
     pub(crate) async fn build_stream_job(
         &self,
         stream_ctx: StreamContext,
-        stream_job: &StreamingJob,
+        mut stream_job: StreamingJob,
         fragment_graph: StreamFragmentGraph,
         affected_table_replace_info: Option<(StreamingJob, StreamFragmentGraph)>,
     ) -> MetaResult<(CreateStreamingJobContext, TableFragments)> {
@@ -1553,7 +1573,7 @@ impl DdlController {
         // 1. Resolve the upstream fragments, extend the fragment graph to a complete graph that
         // contains all information needed for building the actor graph.
 
-        let upstream_root_fragments = self
+        let (upstream_root_fragments, existing_actor_location) = self
             .metadata_manager
             .get_upstream_root_fragments(fragment_graph.dependent_table_ids())
             .await?;
@@ -1571,7 +1591,8 @@ impl DdlController {
         let complete_graph = CompleteStreamFragmentGraph::with_upstreams(
             fragment_graph,
             upstream_root_fragments,
-            stream_job.into(),
+            existing_actor_location,
+            (&stream_job).into(),
         )?;
 
         // 2. Build the actor graph.
@@ -1589,7 +1610,7 @@ impl DdlController {
             dispatchers,
             merge_updates,
         } = actor_graph_builder
-            .generate_graph(&self.env, stream_job, expr_context)
+            .generate_graph(&self.env, &stream_job, expr_context)
             .await?;
         assert!(merge_updates.is_empty());
 
@@ -1614,7 +1635,7 @@ impl DdlController {
 
         let replace_table_job_info = match affected_table_replace_info {
             Some((streaming_job, fragment_graph)) => {
-                let StreamingJob::Sink(s, _) = stream_job else {
+                let StreamingJob::Sink(s, target_table) = &mut stream_job else {
                     bail!("additional replace table event only occurs when sinking into table");
                 };
 
@@ -1651,6 +1672,13 @@ impl DdlController {
                         fragment_graph,
                     )
                     .await?;
+                // When sinking into table occurs, some variables of the target table may be modified,
+                // such as `fragment_id` being altered by `prepare_replace_table`.
+                // At this point, it’s necessary to update the table info carried with the sink.
+                must_match!(&streaming_job, StreamingJob::Table(source, table, _) => {
+                    // The StreamingJob in ReplaceTableInfo must be StreamingJob::Table
+                    *target_table = Some((table.clone(), source.clone()));
+                });
 
                 Some((streaming_job, context, table_fragments))
             }
@@ -1666,7 +1694,8 @@ impl DdlController {
             definition: stream_job.definition(),
             mv_table_id: stream_job.mv_table(),
             create_type: stream_job.create_type(),
-            ddl_type: stream_job.into(),
+            ddl_type: (&stream_job).into(),
+            streaming_job: stream_job,
             replace_table_job_info,
             option: CreateStreamingJobOption {},
         };
@@ -1675,7 +1704,7 @@ impl DdlController {
         let creating_tables = ctx
             .internal_tables()
             .into_iter()
-            .chain(stream_job.table().cloned())
+            .chain(ctx.streaming_job.table().cloned())
             .collect_vec();
 
         if let MetadataManager::V1(mgr) = &self.metadata_manager {
@@ -1814,6 +1843,7 @@ impl DdlController {
         let fragment_graph = self
             .prepare_replace_table(mgr.catalog_manager.clone(), &mut stream_job, fragment_graph)
             .await?;
+
         let dummy_id = self
             .env
             .id_gen_manager()
@@ -1821,8 +1851,10 @@ impl DdlController {
             .generate::<{ IdCategory::Table }>()
             .await? as u32;
 
+        let mut updated_sink_catalogs = vec![];
+
         let result: MetaResult<()> = try {
-            let (ctx, table_fragments) = self
+            let (mut ctx, mut table_fragments) = self
                 .build_replace_table(
                     stream_ctx,
                     &stream_job,
@@ -1832,6 +1864,62 @@ impl DdlController {
                 )
                 .await?;
 
+            let StreamingJob::Table(_, table, _) = &stream_job else {
+                unreachable!("unexpected job: {stream_job:?}");
+            };
+
+            let mut union_fragment_id = None;
+
+            for (fragment_id, fragment) in &mut table_fragments.fragments {
+                for actor in &mut fragment.actors {
+                    if let Some(node) = &mut actor.nodes {
+                        visit_stream_node(node, |body| {
+                            if let NodeBody::Union(_) = body {
+                                if let Some(union_fragment_id) = union_fragment_id.as_mut() {
+                                    // The union fragment should be unique.
+                                    assert_eq!(*union_fragment_id, *fragment_id);
+                                } else {
+                                    union_fragment_id = Some(*fragment_id);
+                                }
+                            }
+                        })
+                    };
+                }
+            }
+
+            let target_fragment_id =
+                union_fragment_id.expect("fragment of placeholder merger not found");
+
+            let catalogs = self
+                .metadata_manager
+                .get_sink_catalog_by_ids(&table.incoming_sinks)
+                .await?;
+
+            for sink in catalogs {
+                let sink_id = &sink.id;
+
+                let sink_table_fragments = self
+                    .metadata_manager
+                    .get_job_fragments_by_id(&risingwave_common::catalog::TableId::new(*sink_id))
+                    .await?;
+
+                let sink_fragment = sink_table_fragments.sink_fragment().unwrap();
+
+                Self::inject_replace_table_plan_for_sink(
+                    Some(*sink_id),
+                    &sink_fragment,
+                    table,
+                    &mut ctx,
+                    &mut table_fragments,
+                    target_fragment_id,
+                    Some(&sink.unique_identity()),
+                );
+
+                if sink.original_target_columns.is_empty() {
+                    updated_sink_catalogs.push(sink.id);
+                }
+            }
+
             // Add table fragments to meta store with state: `State::Initial`.
             mgr.fragment_manager
                 .start_create_table_fragments(table_fragments.clone())
@@ -1850,6 +1938,7 @@ impl DdlController {
                     table_col_index_mapping,
                     None,
                     None,
+                    updated_sink_catalogs,
                 )
                 .await
             }
@@ -1922,8 +2011,20 @@ impl DdlController {
             .mview_fragment()
             .expect("mview fragment not found");
 
+        let ddl_type = DdlType::from(stream_job);
+        let DdlType::Table(table_job_type) = &ddl_type else {
+            bail!(
+                "only support replacing table streaming job, ddl_type: {:?}",
+                ddl_type
+            )
+        };
+
         // Map the column indices in the dispatchers with the given mapping.
-        let downstream_fragments = self.metadata_manager.get_downstream_chain_fragments(id).await?
+        let (downstream_fragments, downstream_actor_location) = self
+            .metadata_manager
+            .get_downstream_chain_fragments(id)
+            .await?;
+        let downstream_fragments = downstream_fragments
             .into_iter()
             .map(|(d, f)|
                 if let Some(mapping) = &table_col_index_mapping {
@@ -1939,12 +2040,37 @@ impl DdlController {
                 )
             })?;
 
-        let complete_graph = CompleteStreamFragmentGraph::with_downstreams(
-            fragment_graph,
-            original_table_fragment.fragment_id,
-            downstream_fragments,
-            stream_job.into(),
-        )?;
+        // build complete graph based on the table job type
+        let complete_graph = match table_job_type {
+            TableJobType::General => CompleteStreamFragmentGraph::with_downstreams(
+                fragment_graph,
+                original_table_fragment.fragment_id,
+                downstream_fragments,
+                downstream_actor_location,
+                ddl_type,
+            )?,
+
+            TableJobType::SharedCdcSource => {
+                // get the upstream fragment which should be the cdc source
+                let (upstream_root_fragments, upstream_actor_location) = self
+                    .metadata_manager
+                    .get_upstream_root_fragments(fragment_graph.dependent_table_ids())
+                    .await?;
+
+                CompleteStreamFragmentGraph::with_upstreams_and_downstreams(
+                    fragment_graph,
+                    upstream_root_fragments,
+                    upstream_actor_location,
+                    original_table_fragment.fragment_id,
+                    downstream_fragments,
+                    downstream_actor_location,
+                    ddl_type,
+                )?
+            }
+            TableJobType::Unspecified => {
+                unreachable!()
+            }
+        };
 
         // 2. Build the actor graph.
         let cluster_info = self.metadata_manager.get_streaming_cluster_info().await?;
@@ -1964,7 +2090,11 @@ impl DdlController {
         } = actor_graph_builder
             .generate_graph(&self.env, stream_job, expr_context)
             .await?;
-        assert!(dispatchers.is_empty());
+
+        // general table job type does not have upstream job, so the dispatchers should be empty
+        if matches!(table_job_type, TableJobType::General) {
+            assert!(dispatchers.is_empty());
+        }
 
         // 3. Build the table fragments structure that will be persisted in the stream manager, and
         // the context that contains all information needed for building the actors on the compute
@@ -1983,6 +2113,8 @@ impl DdlController {
             dispatchers,
             building_locations,
             existing_locations,
+            streaming_job: stream_job.clone(),
+            dummy_id: dummy_table_id,
         };
 
         Ok((ctx, table_fragments))
@@ -1995,6 +2127,7 @@ impl DdlController {
         table_col_index_mapping: Option<ColIndexMapping>,
         creating_sink_id: Option<SinkId>,
         dropping_sink_id: Option<SinkId>,
+        updated_sink_ids: Vec<SinkId>,
     ) -> MetaResult<NotificationVersion> {
         let StreamingJob::Table(source, table, ..) = stream_job else {
             unreachable!("unexpected job: {stream_job:?}")
@@ -2007,6 +2140,7 @@ impl DdlController {
                 table_col_index_mapping,
                 creating_sink_id,
                 dropping_sink_id,
+                updated_sink_ids,
             )
             .await
     }
diff --git a/src/meta/src/rpc/ddl_controller_v2.rs b/src/meta/src/rpc/ddl_controller_v2.rs
index 0dabc9b19022d..c097fa5acb5c6 100644
--- a/src/meta/src/rpc/ddl_controller_v2.rs
+++ b/src/meta/src/rpc/ddl_controller_v2.rs
@@ -14,7 +14,7 @@
 
 use itertools::Itertools;
 use risingwave_common::util::column_index_mapping::ColIndexMapping;
-use risingwave_common::util::stream_graph_visitor::visit_fragment;
+use risingwave_common::util::stream_graph_visitor::{visit_fragment, visit_stream_node};
 use risingwave_meta_model_v2::object::ObjectType;
 use risingwave_meta_model_v2::ObjectId;
 use risingwave_pb::catalog::CreateType;
@@ -81,12 +81,20 @@ impl DdlController {
             .unwrap();
         let _reschedule_job_lock = self.stream_manager.reschedule_lock_read_guard().await;
 
+        let id = streaming_job.id();
+        let name = streaming_job.name();
+        let definition = streaming_job.definition();
+        let source_id = match &streaming_job {
+            StreamingJob::Table(Some(src), _, _) | StreamingJob::Source(src) => Some(src.id),
+            _ => None,
+        };
+
         // create streaming job.
         match self
             .create_streaming_job_inner_v2(
                 mgr,
                 ctx,
-                &mut streaming_job,
+                streaming_job,
                 fragment_graph,
                 affected_table_replace_info,
             )
@@ -94,11 +102,11 @@ impl DdlController {
         {
             Ok(version) => Ok(version),
             Err(err) => {
-                tracing::error!(id = job_id, error = ?err.as_report(), "failed to create streaming job");
+                tracing::error!(id = job_id, error = %err.as_report(), "failed to create streaming job");
                 let event = risingwave_pb::meta::event_log::EventCreateStreamJobFail {
-                    id: streaming_job.id(),
-                    name: streaming_job.name(),
-                    definition: streaming_job.definition(),
+                    id,
+                    name,
+                    definition,
                     error: err.as_report().to_string(),
                 };
                 self.env.event_log_manager_ref().add_event_logs(vec![
@@ -110,11 +118,10 @@ impl DdlController {
                     .await?;
                 if aborted {
                     tracing::warn!(id = job_id, "aborted streaming job");
-                    match &streaming_job {
-                        StreamingJob::Table(Some(src), _, _) | StreamingJob::Source(src) => {
-                            self.source_manager.unregister_sources(vec![src.id]).await;
-                        }
-                        _ => {}
+                    if let Some(source_id) = source_id {
+                        self.source_manager
+                            .unregister_sources(vec![source_id])
+                            .await;
                     }
                 }
                 Err(err)
@@ -126,12 +133,12 @@ impl DdlController {
         &self,
         mgr: &MetadataManagerV2,
         ctx: StreamContext,
-        streaming_job: &mut StreamingJob,
+        mut streaming_job: StreamingJob,
         fragment_graph: StreamFragmentGraphProto,
         affected_table_replace_info: Option<ReplaceTableInfo>,
     ) -> MetaResult<NotificationVersion> {
         let mut fragment_graph =
-            StreamFragmentGraph::new(&self.env, fragment_graph, streaming_job).await?;
+            StreamFragmentGraph::new(&self.env, fragment_graph, &streaming_job).await?;
         streaming_job.set_table_fragment_id(fragment_graph.table_fragment_id());
         streaming_job.set_dml_fragment_id(fragment_graph.dml_fragment_id());
 
@@ -173,6 +180,8 @@ impl DdlController {
             )
             .await?;
 
+        let streaming_job = &ctx.streaming_job;
+
         match streaming_job {
             StreamingJob::Table(None, table, TableJobType::SharedCdcSource) => {
                 Self::validate_cdc_table(table, &table_fragments).await?;
@@ -181,13 +190,7 @@ impl DdlController {
                 // Register the source on the connector node.
                 self.source_manager.register_source(source).await?;
             }
-            StreamingJob::Sink(sink, target_table) => {
-                if let Some((StreamingJob::Table(source, table, _), ..)) =
-                    &ctx.replace_table_job_info
-                {
-                    *target_table = Some((table.clone(), source.clone()));
-                }
-
+            StreamingJob::Sink(sink, _) => {
                 // Validate the sink on the connector node.
                 validate_sink(sink).await?;
             }
@@ -204,50 +207,25 @@ impl DdlController {
 
         // create streaming jobs.
         let stream_job_id = streaming_job.id();
-        match (streaming_job.create_type(), streaming_job) {
+        match (streaming_job.create_type(), &streaming_job) {
             (CreateType::Unspecified, _)
             | (CreateType::Foreground, _)
             // FIXME(kwannoel): Unify background stream's creation path with MV below.
             | (CreateType::Background, StreamingJob::Sink(_, _)) => {
-                let replace_table_job_info = ctx.replace_table_job_info.as_ref().map(
-                    |(streaming_job, ctx, table_fragments)| {
-                        (
-                            streaming_job.clone(),
-                            ctx.merge_updates.clone(),
-                            table_fragments.table_id().table_id(),
-                        )
-                    },
-                );
-
-                self.stream_manager
+                let version = self.stream_manager
                     .create_streaming_job(table_fragments, ctx)
                     .await?;
-
-                let version = mgr
-                    .catalog_controller
-                    .finish_streaming_job(stream_job_id as _, replace_table_job_info)
-                    .await?;
-
                 Ok(version)
             }
             (CreateType::Background, _) => {
                 let ctrl = self.clone();
-                let mgr = mgr.clone();
                 let fut = async move {
-                    let result = ctrl
+                    let _ = ctrl
                         .stream_manager
                         .create_streaming_job(table_fragments, ctx)
                         .await.inspect_err(|err| {
                             tracing::error!(id = stream_job_id, error = ?err.as_report(), "failed to create background streaming job");
                         });
-                    if result.is_ok() {
-                        let _ = mgr
-                            .catalog_controller
-                            .finish_streaming_job(stream_job_id as _, None)
-                            .await.inspect_err(|err| {
-                                tracing::error!(id = stream_job_id, error = ?err.as_report(), "failed to finish background streaming job");
-                            });
-                    }
                 };
                 tokio::spawn(fut);
                 Ok(IGNORED_NOTIFICATION_VERSION)
@@ -359,6 +337,7 @@ impl DdlController {
                             None,
                             None,
                             Some(sink_id),
+                            vec![],
                         )
                         .await?;
                     Ok(version)
@@ -465,8 +444,10 @@ impl DdlController {
             .await?;
 
         tracing::debug!(id = streaming_job.id(), "building replace streaming job");
+        let mut updated_sink_catalogs = vec![];
+
         let result: MetaResult<Vec<PbMergeUpdate>> = try {
-            let (ctx, table_fragments) = self
+            let (mut ctx, mut table_fragments) = self
                 .build_replace_table(
                     ctx,
                     &streaming_job,
@@ -475,6 +456,59 @@ impl DdlController {
                     dummy_id as _,
                 )
                 .await?;
+
+            let mut union_fragment_id = None;
+
+            for (fragment_id, fragment) in &mut table_fragments.fragments {
+                for actor in &mut fragment.actors {
+                    if let Some(node) = &mut actor.nodes {
+                        visit_stream_node(node, |body| {
+                            if let NodeBody::Union(_) = body {
+                                if let Some(union_fragment_id) = union_fragment_id.as_mut() {
+                                    // The union fragment should be unique.
+                                    assert_eq!(*union_fragment_id, *fragment_id);
+                                } else {
+                                    union_fragment_id = Some(*fragment_id);
+                                }
+                            }
+                        })
+                    };
+                }
+            }
+
+            let target_fragment_id =
+                union_fragment_id.expect("fragment of placeholder merger not found");
+
+            let catalogs = self
+                .metadata_manager
+                .get_sink_catalog_by_ids(&table.incoming_sinks)
+                .await?;
+
+            for sink in catalogs {
+                let sink_id = &sink.id;
+
+                let sink_table_fragments = self
+                    .metadata_manager
+                    .get_job_fragments_by_id(&risingwave_common::catalog::TableId::new(*sink_id))
+                    .await?;
+
+                let sink_fragment = sink_table_fragments.sink_fragment().unwrap();
+
+                Self::inject_replace_table_plan_for_sink(
+                    Some(*sink_id),
+                    &sink_fragment,
+                    table,
+                    &mut ctx,
+                    &mut table_fragments,
+                    target_fragment_id,
+                    Some(&sink.unique_identity()),
+                );
+
+                if sink.original_target_columns.is_empty() {
+                    updated_sink_catalogs.push(sink.id);
+                }
+            }
+
             let merge_updates = ctx.merge_updates.clone();
 
             mgr.catalog_controller
@@ -498,6 +532,7 @@ impl DdlController {
                         table_col_index_mapping,
                         None,
                         None,
+                        updated_sink_catalogs,
                     )
                     .await?;
                 Ok(version)
diff --git a/src/meta/src/rpc/election/dummy.rs b/src/meta/src/rpc/election/dummy.rs
new file mode 100644
index 0000000000000..567958dd08600
--- /dev/null
+++ b/src/meta/src/rpc/election/dummy.rs
@@ -0,0 +1,73 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use tokio::sync::watch::{self, Receiver, Sender};
+
+use crate::{ElectionClient, ElectionMember, MetaResult};
+
+/// A dummy implementation of [`ElectionClient`] for scenarios where only one meta node is running,
+/// typically for testing purposes such as an in-memory meta store.
+///
+/// This can be used to unify the code paths no matter there's HA or not.
+pub struct DummyElectionClient {
+    id: String,
+
+    /// A dummy watcher that never changes, indicating we are always the leader.
+    dummy_watcher: Sender<bool>,
+}
+
+impl DummyElectionClient {
+    pub fn new(id: String) -> Self {
+        Self {
+            id,
+            dummy_watcher: watch::channel(true).0,
+        }
+    }
+
+    fn self_member(&self) -> ElectionMember {
+        ElectionMember {
+            id: self.id.clone(),
+            is_leader: true,
+        }
+    }
+}
+
+#[async_trait::async_trait]
+impl ElectionClient for DummyElectionClient {
+    fn id(&self) -> MetaResult<String> {
+        Ok(self.id.clone())
+    }
+
+    async fn run_once(&self, _ttl: i64, mut stop: Receiver<()>) -> MetaResult<()> {
+        // Only exit when the stop signal is received.
+        let _ = stop.changed().await;
+        Ok(())
+    }
+
+    fn subscribe(&self) -> Receiver<bool> {
+        self.dummy_watcher.subscribe()
+    }
+
+    async fn leader(&self) -> MetaResult<Option<ElectionMember>> {
+        Ok(Some(self.self_member()))
+    }
+
+    async fn get_members(&self) -> MetaResult<Vec<ElectionMember>> {
+        Ok(vec![self.self_member()])
+    }
+
+    fn is_leader(&self) -> bool {
+        true
+    }
+}
diff --git a/src/meta/src/rpc/election/etcd.rs b/src/meta/src/rpc/election/etcd.rs
index 96b16f537356e..6834591764360 100644
--- a/src/meta/src/rpc/election/etcd.rs
+++ b/src/meta/src/rpc/election/etcd.rs
@@ -36,7 +36,7 @@ pub struct EtcdElectionClient {
 
 #[async_trait::async_trait]
 impl ElectionClient for EtcdElectionClient {
-    async fn is_leader(&self) -> bool {
+    fn is_leader(&self) -> bool {
         *self.is_leader_sender.borrow()
     }
 
@@ -404,7 +404,7 @@ mod tests {
 
         let leader = new_followers.into_iter().next().unwrap();
 
-        assert!(leader.1.is_leader().await);
+        assert!(leader.1.is_leader());
     }
 
     #[tokio::test]
@@ -434,7 +434,7 @@ mod tests {
         let mut leaders = vec![];
         let mut followers = vec![];
         for (sender, client) in clients {
-            if client.is_leader().await {
+            if client.is_leader() {
                 leaders.push((sender, client));
             } else {
                 followers.push((sender, client));
@@ -476,7 +476,7 @@ mod tests {
         }
 
         for client in &clients {
-            assert!(!client.1.is_leader().await);
+            assert!(!client.1.is_leader());
         }
 
         for (stop_sender, client) in &clients {
diff --git a/src/meta/src/rpc/election/mod.rs b/src/meta/src/rpc/election/mod.rs
index 0c65d497b677e..9b34d19ce2244 100644
--- a/src/meta/src/rpc/election/mod.rs
+++ b/src/meta/src/rpc/election/mod.rs
@@ -11,6 +11,8 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+
+pub mod dummy;
 pub mod etcd;
 pub mod sql;
 
@@ -34,9 +36,12 @@ pub trait ElectionClient: Send + Sync + 'static {
     }
 
     fn id(&self) -> MetaResult<String>;
+    /// Run the long-running election process.
+    ///
+    /// Returns when the leader status is lost, or the stop signal is received.
     async fn run_once(&self, ttl: i64, stop: Receiver<()>) -> MetaResult<()>;
     fn subscribe(&self) -> Receiver<bool>;
     async fn leader(&self) -> MetaResult<Option<ElectionMember>>;
     async fn get_members(&self) -> MetaResult<Vec<ElectionMember>>;
-    async fn is_leader(&self) -> bool;
+    fn is_leader(&self) -> bool;
 }
diff --git a/src/meta/src/rpc/election/sql.rs b/src/meta/src/rpc/election/sql.rs
index 9ec5bd199cf76..62694aaa3ded3 100644
--- a/src/meta/src/rpc/election/sql.rs
+++ b/src/meta/src/rpc/election/sql.rs
@@ -781,7 +781,7 @@ where
             .collect())
     }
 
-    async fn is_leader(&self) -> bool {
+    fn is_leader(&self) -> bool {
         *self.is_leader_sender.borrow()
     }
 }
@@ -842,7 +842,7 @@ mod tests {
         loop {
             receiver.changed().await.unwrap();
             if *receiver.borrow() {
-                assert!(sql_election_client.is_leader().await);
+                assert!(sql_election_client.is_leader());
                 break;
             }
         }
@@ -874,7 +874,7 @@ mod tests {
         let mut is_leaders = vec![];
 
         for client in clients {
-            is_leaders.push(client.is_leader().await);
+            is_leaders.push(client.is_leader());
         }
 
         assert!(is_leaders.iter().filter(|&x| *x).count() <= 1);
diff --git a/src/meta/src/rpc/metrics.rs b/src/meta/src/rpc/metrics.rs
index 28520720e98fe..79d34c6904be2 100644
--- a/src/meta/src/rpc/metrics.rs
+++ b/src/meta/src/rpc/metrics.rs
@@ -29,6 +29,7 @@ use risingwave_common::metrics::LabelGuardedIntGaugeVec;
 use risingwave_common::monitor::GLOBAL_METRICS_REGISTRY;
 use risingwave_common::register_guarded_int_gauge_vec_with_registry;
 use risingwave_connector::source::monitor::EnumeratorMetrics as SourceEnumeratorMetrics;
+use risingwave_meta_model_v2::WorkerId;
 use risingwave_object_store::object::object_metrics::{
     ObjectStoreMetrics, GLOBAL_OBJECT_STORE_METRICS,
 };
@@ -717,7 +718,7 @@ impl Default for MetaMetrics {
 
 pub fn start_worker_info_monitor(
     metadata_manager: MetadataManager,
-    election_client: Option<ElectionClientRef>,
+    election_client: ElectionClientRef,
     interval: Duration,
     meta_metrics: Arc<MetaMetrics>,
 ) -> (JoinHandle<()>, Sender<()>) {
@@ -754,9 +755,7 @@ pub fn start_worker_info_monitor(
                     .with_label_values(&[(worker_type.as_str_name())])
                     .set(worker_num as i64);
             }
-            if let Some(client) = &election_client
-                && let Ok(meta_members) = client.get_members().await
-            {
+            if let Ok(meta_members) = election_client.get_members().await {
                 meta_metrics
                     .worker_num
                     .with_label_values(&[WorkerType::Meta.as_str_name()])
@@ -820,17 +819,14 @@ pub async fn refresh_fragment_info_metrics_v2(
         }
     };
 
-    let pu_addr_mapping: HashMap<u32, String> = worker_nodes
+    let worker_addr_mapping: HashMap<WorkerId, String> = worker_nodes
         .into_iter()
-        .flat_map(|worker_node| {
+        .map(|worker_node| {
             let addr = match worker_node.host {
                 Some(host) => format!("{}:{}", host.host, host.port),
                 None => "".to_owned(),
             };
-            worker_node
-                .parallel_units
-                .into_iter()
-                .map(move |pu| (pu.id, addr.clone()))
+            (worker_node.id as WorkerId, addr)
         })
         .collect();
     let table_compaction_group_id_mapping = hummock_manager
@@ -847,7 +843,7 @@ pub async fn refresh_fragment_info_metrics_v2(
         let fragment_id_str = actor_location.fragment_id.to_string();
         // Report a dummy gauge metrics with (fragment id, actor id, node
         // address) as its label
-        if let Some(address) = pu_addr_mapping.get(&(actor_location.parallel_unit_id as u32)) {
+        if let Some(address) = worker_addr_mapping.get(&actor_location.worker_id) {
             meta_metrics
                 .actor_info
                 .with_label_values(&[&actor_id_str, &fragment_id_str, address])
@@ -970,17 +966,11 @@ pub fn start_fragment_info_monitor(
                         if let Some(actor_status) =
                             table_fragments.actor_status.get(&actor.actor_id)
                         {
-                            if let Some(pu) = &actor_status.parallel_unit {
-                                if let Some(address) = workers.get(&pu.worker_node_id) {
-                                    meta_metrics
-                                        .actor_info
-                                        .with_label_values(&[
-                                            &actor_id_str,
-                                            &fragment_id_str,
-                                            address,
-                                        ])
-                                        .set(1);
-                                }
+                            if let Some(address) = workers.get(&actor_status.worker_id()) {
+                                meta_metrics
+                                    .actor_info
+                                    .with_label_values(&[&actor_id_str, &fragment_id_str, address])
+                                    .set(1);
                             }
                         }
 
diff --git a/src/meta/src/storage/wrapped_etcd_client.rs b/src/meta/src/storage/wrapped_etcd_client.rs
index 33d8cd57716d0..fead6b22cac35 100644
--- a/src/meta/src/storage/wrapped_etcd_client.rs
+++ b/src/meta/src/storage/wrapped_etcd_client.rs
@@ -90,6 +90,22 @@ impl EtcdRefreshClient {
     }
 }
 
+#[easy_ext::ext(KvClientUnlimitedExt)]
+impl etcd_client::Client {
+    /// Gets a KV client with message size limit removed.
+    ///
+    /// Check the background at <https://github.com/risingwavelabs/risingwave/pull/11277>.
+    fn kv_client_unlimited(&self) -> etcd_client::KvClient {
+        #[cfg(madsim)]
+        return self.kv_client();
+
+        #[cfg(not(madsim))]
+        self.kv_client()
+            .max_decoding_message_size(usize::MAX)
+            .max_encoding_message_size(usize::MAX)
+    }
+}
+
 macro_rules! impl_etcd_client_command_proxy {
     ($func:ident, $client:ident, ($($arg:ident : $sig:ty),+), $result:ty) => {
         impl WrappedEtcdClient {
@@ -131,10 +147,10 @@ macro_rules! impl_etcd_client_command_proxy {
     }
 }
 
-impl_etcd_client_command_proxy!(get, kv_client, (key: impl Into<Vec<u8>> + Clone, opts: Option<GetOptions>), GetResponse);
-impl_etcd_client_command_proxy!(put, kv_client, (key: impl Into<Vec<u8>> + Clone, value: impl Into<Vec<u8>> + Clone, opts: Option<PutOptions>), PutResponse);
-impl_etcd_client_command_proxy!(delete, kv_client, (key: impl Into<Vec<u8>> + Clone, opts: Option<DeleteOptions>), DeleteResponse);
-impl_etcd_client_command_proxy!(txn, kv_client, (txn: Txn), TxnResponse);
+impl_etcd_client_command_proxy!(get, kv_client_unlimited, (key: impl Into<Vec<u8>> + Clone, opts: Option<GetOptions>), GetResponse);
+impl_etcd_client_command_proxy!(put, kv_client_unlimited, (key: impl Into<Vec<u8>> + Clone, value: impl Into<Vec<u8>> + Clone, opts: Option<PutOptions>), PutResponse);
+impl_etcd_client_command_proxy!(delete, kv_client_unlimited, (key: impl Into<Vec<u8>> + Clone, opts: Option<DeleteOptions>), DeleteResponse);
+impl_etcd_client_command_proxy!(txn, kv_client_unlimited, (txn: Txn), TxnResponse);
 impl_etcd_client_command_proxy!(
     grant,
     lease_client,
diff --git a/src/meta/src/stream/mod.rs b/src/meta/src/stream/mod.rs
index 57ca9896257f3..ec542ed8c008b 100644
--- a/src/meta/src/stream/mod.rs
+++ b/src/meta/src/stream/mod.rs
@@ -36,16 +36,16 @@ pub use stream_manager::*;
 pub(crate) fn to_build_actor_info(
     actor: StreamActor,
     subscriptions: &HashMap<TableId, HashMap<u32, u64>>,
-    fragment_table_id: TableId,
+    subscription_depend_table_id: TableId,
 ) -> BuildActorInfo {
     BuildActorInfo {
         actor: Some(actor),
         related_subscriptions: subscriptions
-            .get(&fragment_table_id)
+            .get(&subscription_depend_table_id)
             .into_iter()
             .map(|subscriptions| {
                 (
-                    fragment_table_id.table_id,
+                    subscription_depend_table_id.table_id,
                     SubscriptionIds {
                         subscription_ids: subscriptions.keys().cloned().collect(),
                     },
diff --git a/src/meta/src/stream/scale.rs b/src/meta/src/stream/scale.rs
index 0b7a711b38643..f61789c8f9372 100644
--- a/src/meta/src/stream/scale.rs
+++ b/src/meta/src/stream/scale.rs
@@ -26,15 +26,12 @@ use itertools::Itertools;
 use num_integer::Integer;
 use num_traits::abs;
 use risingwave_common::bail;
-use risingwave_common::buffer::{Bitmap, BitmapBuilder};
+use risingwave_common::bitmap::{Bitmap, BitmapBuilder};
 use risingwave_common::catalog::TableId;
-use risingwave_common::hash::{ActorMapping, ParallelUnitId, VirtualNode};
+use risingwave_common::hash::{ActorMapping, VirtualNode};
 use risingwave_common::util::iter_util::ZipEqDebug;
 use risingwave_meta_model_v2::StreamingParallelism;
-use risingwave_pb::common::{
-    ActorInfo, Buffer, ParallelUnit, ParallelUnitMapping, WorkerNode, WorkerType,
-};
-use risingwave_pb::meta::get_reschedule_plan_request::{Policy, StableResizePolicy};
+use risingwave_pb::common::{ActorInfo, Buffer, PbActorLocation, WorkerNode, WorkerType};
 use risingwave_pb::meta::subscribe_response::{Info, Operation};
 use risingwave_pb::meta::table_fragments::actor_status::ActorState;
 use risingwave_pb::meta::table_fragments::fragment::{
@@ -53,7 +50,6 @@ use tokio::sync::oneshot::Receiver;
 use tokio::sync::{oneshot, RwLock, RwLockReadGuard, RwLockWriteGuard};
 use tokio::task::JoinHandle;
 use tokio::time::{Instant, MissedTickBehavior};
-use tracing::warn;
 
 use crate::barrier::{Command, Reschedule, StreamRpcManager};
 use crate::manager::{
@@ -110,16 +106,14 @@ impl TableRevision {
 }
 
 #[derive(Debug, Clone, Eq, PartialEq)]
-pub struct ParallelUnitReschedule {
-    pub added_parallel_units: BTreeSet<ParallelUnitId>,
-    pub removed_parallel_units: BTreeSet<ParallelUnitId>,
+pub struct WorkerReschedule {
+    pub worker_actor_diff: BTreeMap<WorkerId, isize>,
 }
 
 pub struct CustomFragmentInfo {
     pub fragment_id: u32,
     pub fragment_type_mask: u32,
     pub distribution_type: PbFragmentDistributionType,
-    pub vnode_mapping: Option<ParallelUnitMapping>,
     pub state_table_ids: Vec<u32>,
     pub upstream_fragment_ids: Vec<u32>,
     pub actor_template: PbStreamActor,
@@ -162,7 +156,6 @@ impl From<&PbFragment> for CustomFragmentInfo {
             fragment_id: fragment.fragment_id,
             fragment_type_mask: fragment.fragment_type_mask,
             distribution_type: fragment.distribution_type(),
-            vnode_mapping: fragment.vnode_mapping.clone(),
             state_table_ids: fragment.state_table_ids.clone(),
             upstream_fragment_ids: fragment.upstream_fragment_ids.clone(),
             actor_template: fragment
@@ -186,12 +179,10 @@ impl CustomFragmentInfo {
 }
 
 pub struct RescheduleContext {
-    /// Index used to map `ParallelUnitId` to `WorkerId`
-    parallel_unit_id_to_worker_id: BTreeMap<ParallelUnitId, WorkerId>,
     /// Meta information for all Actors
     actor_map: HashMap<ActorId, CustomActorInfo>,
     /// Status of all Actors, used to find the location of the `Actor`
-    actor_status: BTreeMap<ActorId, ActorStatus>,
+    actor_status: BTreeMap<ActorId, WorkerId>,
     /// Meta information of all `Fragment`, used to find the `Fragment`'s `Actor`
     fragment_map: HashMap<FragmentId, CustomFragmentInfo>,
     /// Indexes for all `Worker`s
@@ -209,27 +200,11 @@ pub struct RescheduleContext {
 }
 
 impl RescheduleContext {
-    fn actor_id_to_parallel_unit(&self, actor_id: &ActorId) -> MetaResult<&ParallelUnit> {
+    fn actor_id_to_worker_id(&self, actor_id: &ActorId) -> MetaResult<WorkerId> {
         self.actor_status
             .get(actor_id)
-            .and_then(|actor_status| actor_status.parallel_unit.as_ref())
-            .ok_or_else(|| anyhow!("could not found parallel unit for actor {}", actor_id).into())
-    }
-
-    fn parallel_unit_id_to_worker(
-        &self,
-        parallel_unit_id: &ParallelUnitId,
-    ) -> MetaResult<&WorkerNode> {
-        self.parallel_unit_id_to_worker_id
-            .get(parallel_unit_id)
-            .and_then(|worker_id| self.worker_nodes.get(worker_id))
-            .ok_or_else(|| {
-                anyhow!(
-                    "could not found Worker for ParallelUint {}",
-                    parallel_unit_id
-                )
-                .into()
-            })
+            .cloned()
+            .ok_or_else(|| anyhow!("could not find worker for actor {}", actor_id).into())
     }
 }
 
@@ -481,7 +456,7 @@ impl ScaleController {
     /// Build the context for rescheduling and do some validation for the request.
     async fn build_reschedule_context(
         &self,
-        reschedule: &mut HashMap<FragmentId, ParallelUnitReschedule>,
+        reschedule: &mut HashMap<FragmentId, WorkerReschedule>,
         options: RescheduleOptions,
         table_parallelisms: Option<&mut HashMap<TableId, TableParallelism>>,
     ) -> MetaResult<RescheduleContext> {
@@ -498,7 +473,7 @@ impl ScaleController {
         }
 
         // Check if we are trying to move a fragment to a node marked as unschedulable
-        let unschedulable_parallel_unit_ids: HashMap<_, _> = worker_nodes
+        let unschedulable_worker_ids: HashSet<_> = worker_nodes
             .values()
             .filter(|w| {
                 w.property
@@ -506,43 +481,27 @@ impl ScaleController {
                     .map(|property| property.is_unschedulable)
                     .unwrap_or(false)
             })
-            .flat_map(|w| {
-                w.parallel_units
-                    .iter()
-                    .map(|parallel_unit| (parallel_unit.id as ParallelUnitId, w.id as WorkerId))
-            })
+            .map(|worker| worker.id)
             .collect();
 
         for (fragment_id, reschedule) in &*reschedule {
-            for parallel_unit_id in &reschedule.added_parallel_units {
-                if let Some(worker_id) = unschedulable_parallel_unit_ids.get(parallel_unit_id) {
+            for (worker_id, change) in &reschedule.worker_actor_diff {
+                if unschedulable_worker_ids.contains(worker_id) && change.is_positive() {
                     bail!(
-                        "unable to move fragment {} to unschedulable parallel unit {} from worker {}",
+                        "unable to move fragment {} to unschedulable worker {}",
                         fragment_id,
-                        parallel_unit_id,
                         worker_id
                     );
                 }
             }
         }
 
-        // Associating ParallelUnit with Worker
-        let parallel_unit_id_to_worker_id: BTreeMap<_, _> = worker_nodes
-            .iter()
-            .flat_map(|(worker_id, worker_node)| {
-                worker_node
-                    .parallel_units
-                    .iter()
-                    .map(move |parallel_unit| (parallel_unit.id as ParallelUnitId, *worker_id))
-            })
-            .collect();
-
         // FIXME: the same as anther place calling `list_table_fragments` in scaling.
         // Index for StreamActor
         let mut actor_map = HashMap::new();
         // Index for Fragment
         let mut fragment_map = HashMap::new();
-        // Index for actor status, including actor's parallel unit
+        // Index for actor status, including actor's worker id
         let mut actor_status = BTreeMap::new();
         let mut fragment_state = HashMap::new();
         let mut fragment_to_table = HashMap::new();
@@ -551,7 +510,7 @@ impl ScaleController {
         fn fulfill_index_by_table_fragments_ref(
             actor_map: &mut HashMap<u32, CustomActorInfo>,
             fragment_map: &mut HashMap<FragmentId, CustomFragmentInfo>,
-            actor_status: &mut BTreeMap<ActorId, ActorStatus>,
+            actor_status: &mut BTreeMap<ActorId, WorkerId>,
             fragment_state: &mut HashMap<FragmentId, State>,
             fragment_to_table: &mut HashMap<FragmentId, TableId>,
             table_fragments: &TableFragments,
@@ -570,7 +529,9 @@ impl ScaleController {
                 fragment_map.insert(*fragment_id, CustomFragmentInfo::from(fragment));
             }
 
-            actor_status.extend(table_fragments.actor_status.clone());
+            for (actor_id, status) in &table_fragments.actor_status {
+                actor_status.insert(*actor_id, status.worker_id());
+            }
 
             fragment_to_table.extend(
                 table_fragments
@@ -668,14 +629,7 @@ impl ScaleController {
 
         let mut stream_source_fragment_ids = HashSet::new();
         let mut no_shuffle_reschedule = HashMap::new();
-        for (
-            fragment_id,
-            ParallelUnitReschedule {
-                added_parallel_units,
-                removed_parallel_units,
-            },
-        ) in &*reschedule
-        {
+        for (fragment_id, WorkerReschedule { worker_actor_diff }) in &*reschedule {
             let fragment = fragment_map
                 .get(fragment_id)
                 .ok_or_else(|| anyhow!("fragment {fragment_id} does not exist"))?;
@@ -726,9 +680,8 @@ impl ScaleController {
 
                     no_shuffle_reschedule.insert(
                         downstream_id,
-                        ParallelUnitReschedule {
-                            added_parallel_units: added_parallel_units.clone(),
-                            removed_parallel_units: removed_parallel_units.clone(),
+                        WorkerReschedule {
+                            worker_actor_diff: worker_actor_diff.clone(),
                         },
                     );
                 }
@@ -742,54 +695,44 @@ impl ScaleController {
             }
 
             // Check if the reschedule plan is valid.
-            let current_parallel_units = fragment
+            let current_worker_ids = fragment
                 .actors
                 .iter()
-                .map(|a| {
-                    actor_status
-                        .get(&a.actor_id)
-                        .unwrap()
-                        .get_parallel_unit()
-                        .unwrap()
-                        .id
-                })
+                .map(|a| actor_status.get(&a.actor_id).cloned().unwrap())
                 .collect::<HashSet<_>>();
-            for removed in removed_parallel_units {
-                if !current_parallel_units.contains(removed) {
+
+            for (removed, change) in worker_actor_diff {
+                if !current_worker_ids.contains(removed) && change.is_negative() {
                     bail!(
-                        "no actor on the parallel unit {} of fragment {}",
+                        "no actor on the worker {} of fragment {}",
                         removed,
                         fragment_id
                     );
                 }
             }
-            for added in added_parallel_units {
-                if !parallel_unit_id_to_worker_id.contains_key(added) {
-                    bail!("parallel unit {} not available", added);
-                }
-                if current_parallel_units.contains(added) && !removed_parallel_units.contains(added)
-                {
-                    bail!(
-                        "parallel unit {} of fragment {} is already in use",
-                        added,
-                        fragment_id
-                    );
-                }
-            }
+
+            let added_actor_count: usize = worker_actor_diff
+                .values()
+                .filter(|change| change.is_positive())
+                .cloned()
+                .map(|change| change as usize)
+                .sum();
+
+            let removed_actor_count: usize = worker_actor_diff
+                .values()
+                .filter(|change| change.is_positive())
+                .cloned()
+                .map(|v| v.unsigned_abs())
+                .sum();
 
             match fragment.distribution_type() {
                 FragmentDistributionType::Hash => {
-                    if current_parallel_units.len() + added_parallel_units.len()
-                        <= removed_parallel_units.len()
-                    {
-                        bail!(
-                            "can't remove all parallel units from fragment {}",
-                            fragment_id
-                        );
+                    if fragment.actors.len() + added_actor_count <= removed_actor_count {
+                        bail!("can't remove all actors from fragment {}", fragment_id);
                     }
                 }
                 FragmentDistributionType::Single => {
-                    if added_parallel_units.len() != removed_parallel_units.len() {
+                    if added_actor_count != removed_actor_count {
                         bail!("single distribution fragment only support migration");
                     }
                 }
@@ -808,7 +751,6 @@ impl ScaleController {
         reschedule.extend(no_shuffle_reschedule.into_iter());
 
         Ok(RescheduleContext {
-            parallel_unit_id_to_worker_id,
             actor_map,
             actor_status,
             fragment_map,
@@ -821,7 +763,7 @@ impl ScaleController {
         })
     }
 
-    async fn create_actors_on_compute_node(
+    pub(crate) async fn create_actors_on_compute_node(
         &self,
         worker_nodes: &HashMap<WorkerId, WorkerNode>,
         actor_infos_to_broadcast: BTreeMap<ActorId, ActorInfo>,
@@ -857,10 +799,20 @@ impl ScaleController {
         Ok(())
     }
 
-    // Results are the generated reschedule plan and the changes that need to be updated to the meta store.
-    pub(crate) async fn prepare_reschedule_command(
+    /// From the high-level [`WorkerReschedule`] to the low-level reschedule plan [`Reschedule`].
+    ///
+    /// Returns `(reschedule_fragment, applied_reschedules)`
+    /// - `reschedule_fragment`: the generated reschedule plan
+    /// - `applied_reschedules`: the changes that need to be updated to the meta store (`pre_apply_reschedules`, only for V1).
+    ///
+    /// In [normal process of scaling](`GlobalStreamManager::reschedule_actors_impl`), we use the returned values to
+    /// build a [`Command::RescheduleFragment`], which will then flows through the barrier mechanism to perform scaling.
+    /// Meta store is updated after the barrier is collected.
+    ///
+    /// During recovery, we don't need the barrier mechanism, and can directly use the returned values to update meta.
+    pub(crate) async fn analyze_reschedule_plan(
         &self,
-        mut reschedules: HashMap<FragmentId, ParallelUnitReschedule>,
+        mut reschedules: HashMap<FragmentId, WorkerReschedule>,
         options: RescheduleOptions,
         table_parallelisms: Option<&mut HashMap<TableId, TableParallelism>>,
     ) -> MetaResult<(
@@ -870,9 +822,12 @@ impl ScaleController {
         let ctx = self
             .build_reschedule_context(&mut reschedules, options, table_parallelisms)
             .await?;
-        // Index of actors to create/remove
-        // Fragment Id => ( Actor Id => Parallel Unit Id )
+        let reschedules = reschedules;
+
+        // Here, the plan for both upstream and downstream of the NO_SHUFFLE Fragment should already have been populated.
 
+        // Index of actors to create/remove
+        // Fragment Id => ( Actor Id => Worker Id )
         let (fragment_actors_to_remove, fragment_actors_to_create) =
             self.arrange_reschedules(&reschedules, &ctx).await?;
 
@@ -886,19 +841,19 @@ impl ScaleController {
 
             let actors_to_create = fragment_actors_to_create
                 .get(fragment_id)
-                .map(|map| map.keys().cloned().collect())
+                .map(|map| map.iter().map(|(actor_id, _)| *actor_id).collect())
                 .unwrap_or_default();
 
             let actors_to_remove = fragment_actors_to_remove
                 .get(fragment_id)
-                .map(|map| map.keys().cloned().collect())
+                .map(|map| map.iter().map(|(actor_id, _)| *actor_id).collect())
                 .unwrap_or_default();
 
             let fragment = ctx.fragment_map.get(fragment_id).unwrap();
 
             match fragment.distribution_type() {
                 FragmentDistributionType::Single => {
-                    // Skip rebalance action for single distribution (always None)
+                    // Skip re-balancing action for single distribution (always None)
                     fragment_actor_bitmap
                         .insert(fragment.fragment_id as FragmentId, Default::default());
                 }
@@ -916,7 +871,7 @@ impl ScaleController {
             }
         }
 
-        // Index for fragment -> { actor -> parallel_unit } after reschedule.
+        // Index for fragment -> { actor -> worker_id } after reschedule.
         // Since we need to organize the upstream and downstream relationships of NoShuffle,
         // we need to organize the actor distribution after a scaling.
         let mut fragment_actors_after_reschedule = HashMap::with_capacity(reschedules.len());
@@ -929,16 +884,13 @@ impl ScaleController {
                         continue;
                     }
                 }
-                let parallel_unit_id = ctx.actor_id_to_parallel_unit(&actor.actor_id)?.id;
-                new_actor_ids.insert(
-                    actor.actor_id as ActorId,
-                    parallel_unit_id as ParallelUnitId,
-                );
+                let worker_id = ctx.actor_id_to_worker_id(&actor.actor_id)?;
+                new_actor_ids.insert(actor.actor_id as ActorId, worker_id);
             }
 
             if let Some(actors_to_create) = fragment_actors_to_create.get(fragment_id) {
-                for (actor_id, parallel_unit_id) in actors_to_create {
-                    new_actor_ids.insert(*actor_id, *parallel_unit_id as ParallelUnitId);
+                for (actor_id, worker_id) in actors_to_create {
+                    new_actor_ids.insert(*actor_id, *worker_id);
                 }
             }
 
@@ -954,20 +906,16 @@ impl ScaleController {
         let fragment_actors_after_reschedule = fragment_actors_after_reschedule;
 
         // In order to maintain consistency with the original structure, the upstream and downstream
-        // actors of NoShuffle need to be in the same parallel unit and hold the same virtual nodes,
-        // so for the actors after the upstream rebalancing, we need to find the parallel
-        // unit corresponding to each actor, and find the downstream actor corresponding to
-        // the parallel unit, and then copy the Bitmap to the corresponding actor. At the
-        // same time, we need to sort out the relationship between upstream and downstream
-        // actors
+        // actors of NoShuffle need to be in the same worker slot and hold the same virtual nodes,
+        // so for the actors after the upstream re-balancing, since we have sorted the actors of the same fragment by id on all workers,
+        // we can identify the corresponding upstream actor with NO_SHUFFLE.
+        // NOTE: There should be more asserts here to ensure correctness.
         fn arrange_no_shuffle_relation(
             ctx: &RescheduleContext,
             fragment_id: &FragmentId,
             upstream_fragment_id: &FragmentId,
-            fragment_actors_after_reschedule: &HashMap<
-                FragmentId,
-                BTreeMap<ActorId, ParallelUnitId>,
-            >,
+            fragment_actors_after_reschedule: &HashMap<FragmentId, BTreeMap<ActorId, WorkerId>>,
+            actor_group_map: &mut HashMap<ActorId, (FragmentId, ActorId)>,
             fragment_updated_bitmap: &mut HashMap<FragmentId, HashMap<ActorId, Bitmap>>,
             no_shuffle_upstream_actor_map: &mut HashMap<ActorId, HashMap<FragmentId, ActorId>>,
             no_shuffle_downstream_actors_map: &mut HashMap<ActorId, HashMap<FragmentId, ActorId>>,
@@ -978,8 +926,39 @@ impl ScaleController {
 
             let fragment = ctx.fragment_map.get(fragment_id).unwrap();
 
+            let upstream_fragment = ctx.fragment_map.get(upstream_fragment_id).unwrap();
+
+            for upstream_actor in &upstream_fragment.actors {
+                for dispatcher in &upstream_actor.dispatcher {
+                    if let DispatcherType::NoShuffle = dispatcher.get_type().unwrap() {
+                        let downstream_actor_id =
+                            *dispatcher.downstream_actor_id.iter().exactly_one().unwrap();
+
+                        // upstream is root
+                        if !ctx
+                            .no_shuffle_target_fragment_ids
+                            .contains(upstream_fragment_id)
+                        {
+                            actor_group_map.insert(
+                                upstream_actor.actor_id,
+                                (upstream_fragment.fragment_id, upstream_actor.actor_id),
+                            );
+                            actor_group_map.insert(
+                                downstream_actor_id,
+                                (upstream_fragment.fragment_id, upstream_actor.actor_id),
+                            );
+                        } else {
+                            let root_actor_id =
+                                *actor_group_map.get(&upstream_actor.actor_id).unwrap();
+
+                            actor_group_map.insert(downstream_actor_id, root_actor_id);
+                        }
+                    }
+                }
+            }
+
             // If the upstream is a Singleton Fragment, there will be no Bitmap changes
-            let mut upstream_fragment_bitmap = fragment_updated_bitmap
+            let upstream_fragment_bitmap = fragment_updated_bitmap
                 .get(upstream_fragment_id)
                 .cloned()
                 .unwrap_or_default();
@@ -989,30 +968,84 @@ impl ScaleController {
                 .cloned()
                 .unwrap();
 
-            let mut parallel_unit_id_to_actor_id = HashMap::new();
-            for (actor_id, parallel_unit_id) in
-                fragment_actors_after_reschedule.get(fragment_id).unwrap()
-            {
-                parallel_unit_id_to_actor_id.insert(*parallel_unit_id, *actor_id);
-            }
+            let fragment_actor_map = fragment_actors_after_reschedule
+                .get(fragment_id)
+                .cloned()
+                .unwrap();
+
+            let mut worker_reverse_index: HashMap<WorkerId, BTreeSet<_>> = HashMap::new();
 
+            // first, find existing actor bitmap, copy them
             let mut fragment_bitmap = HashMap::new();
-            for (upstream_actor_id, parallel_unit_id) in upstream_fragment_actor_map {
-                let actor_id = parallel_unit_id_to_actor_id.get(&parallel_unit_id).unwrap();
 
-                if let Some(bitmap) = upstream_fragment_bitmap.remove(&upstream_actor_id) {
+            for (actor_id, worker_id) in &fragment_actor_map {
+                if let Some((root_fragment, root_actor_id)) = actor_group_map.get(actor_id) {
+                    let root_bitmap = fragment_updated_bitmap
+                        .get(root_fragment)
+                        .expect("root fragment bitmap not found")
+                        .get(root_actor_id)
+                        .cloned()
+                        .expect("root actor bitmap not found");
+
                     // Copy the bitmap
-                    fragment_bitmap.insert(*actor_id, bitmap);
+                    fragment_bitmap.insert(*actor_id, root_bitmap);
+
+                    no_shuffle_upstream_actor_map
+                        .entry(*actor_id as ActorId)
+                        .or_default()
+                        .insert(*upstream_fragment_id, *root_actor_id);
+                    no_shuffle_downstream_actors_map
+                        .entry(*root_actor_id)
+                        .or_default()
+                        .insert(*fragment_id, *actor_id);
+                } else {
+                    worker_reverse_index
+                        .entry(*worker_id)
+                        .or_default()
+                        .insert(*actor_id);
+                }
+            }
+
+            let mut upstream_worker_reverse_index: HashMap<WorkerId, BTreeSet<_>> = HashMap::new();
+
+            for (actor_id, worker_id) in &upstream_fragment_actor_map {
+                if !actor_group_map.contains_key(actor_id) {
+                    upstream_worker_reverse_index
+                        .entry(*worker_id)
+                        .or_default()
+                        .insert(*actor_id);
                 }
+            }
 
-                no_shuffle_upstream_actor_map
-                    .entry(*actor_id as ActorId)
-                    .or_default()
-                    .insert(*upstream_fragment_id, upstream_actor_id);
-                no_shuffle_downstream_actors_map
-                    .entry(upstream_actor_id)
-                    .or_default()
-                    .insert(*fragment_id, *actor_id);
+            // then, find the rest of the actors and copy the bitmap
+            for (worker_id, actor_ids) in worker_reverse_index {
+                let upstream_actor_ids = upstream_worker_reverse_index
+                    .get(&worker_id)
+                    .unwrap()
+                    .clone();
+                assert_eq!(actor_ids.len(), upstream_actor_ids.len());
+
+                for (actor_id, upstream_actor_id) in actor_ids
+                    .into_iter()
+                    .zip_eq_debug(upstream_actor_ids.into_iter())
+                {
+                    let bitmap = upstream_fragment_bitmap
+                        .get(&upstream_actor_id)
+                        .cloned()
+                        .unwrap();
+
+                    // Copy the bitmap
+                    fragment_bitmap.insert(actor_id, bitmap);
+
+                    no_shuffle_upstream_actor_map
+                        .entry(actor_id as ActorId)
+                        .or_default()
+                        .insert(*upstream_fragment_id, upstream_actor_id);
+                    no_shuffle_downstream_actors_map
+                        .entry(upstream_actor_id)
+                        .or_default()
+                        .insert(*fragment_id, actor_id);
+                }
             }
 
             match fragment.distribution_type() {
@@ -1045,6 +1078,7 @@ impl ScaleController {
                         downstream_fragment_id,
                         fragment_id,
                         fragment_actors_after_reschedule,
+                        actor_group_map,
                         fragment_updated_bitmap,
                         no_shuffle_upstream_actor_map,
                         no_shuffle_downstream_actors_map,
@@ -1055,6 +1089,7 @@ impl ScaleController {
 
         let mut no_shuffle_upstream_actor_map = HashMap::new();
         let mut no_shuffle_downstream_actors_map = HashMap::new();
+        let mut actor_group_map = HashMap::new();
         // For all roots in the upstream and downstream dependency trees of NoShuffle, recursively
         // find all correspondences
         for fragment_id in reschedules.keys() {
@@ -1068,6 +1103,7 @@ impl ScaleController {
                             downstream_fragment_id,
                             fragment_id,
                             &fragment_actors_after_reschedule,
+                            &mut actor_group_map,
                             &mut fragment_actor_bitmap,
                             &mut no_shuffle_upstream_actor_map,
                             &mut no_shuffle_downstream_actors_map,
@@ -1077,6 +1113,8 @@ impl ScaleController {
             }
         }
 
+        tracing::debug!("actor group map {:?}", actor_group_map);
+
         let mut new_created_actors = HashMap::new();
         for fragment_id in reschedules.keys() {
             let actors_to_create = fragment_actors_to_create
@@ -1125,7 +1163,7 @@ impl ScaleController {
             // After modification, for newly created actors, both upstream and downstream actor ids
             // have been modified
             let mut actor_infos_to_broadcast = BTreeMap::new();
-            let mut node_actors_to_create: HashMap<WorkerId, Vec<BuildActorInfo>> = HashMap::new();
+            let mut node_actors_to_create: HashMap<WorkerId, Vec<_>> = HashMap::new();
             let mut broadcast_worker_ids = HashSet::new();
 
             let subscriptions: HashMap<_, SubscriptionIds> = self
@@ -1144,16 +1182,15 @@ impl ScaleController {
                 .collect();
 
             for actors_to_create in fragment_actors_to_create.values() {
-                for (new_actor_id, new_parallel_unit_id) in actors_to_create {
+                for (new_actor_id, worker_id) in actors_to_create {
                     let new_actor = new_created_actors.get(new_actor_id).unwrap();
                     for upstream_actor_id in &new_actor.upstream_actor_id {
                         if new_created_actors.contains_key(upstream_actor_id) {
                             continue;
                         }
 
-                        let upstream_worker_id = ctx
-                            .actor_id_to_parallel_unit(upstream_actor_id)?
-                            .worker_node_id;
+                        let upstream_worker_id = ctx.actor_id_to_worker_id(upstream_actor_id)?;
+
                         let upstream_worker =
                             ctx.worker_nodes.get(&upstream_worker_id).with_context(|| {
                                 format!("upstream worker {} not found", upstream_worker_id)
@@ -1177,9 +1214,9 @@ impl ScaleController {
                             if new_created_actors.contains_key(downstream_actor_id) {
                                 continue;
                             }
-                            let downstream_worker_id = ctx
-                                .actor_id_to_parallel_unit(downstream_actor_id)?
-                                .worker_node_id;
+                            let downstream_worker_id =
+                                ctx.actor_id_to_worker_id(downstream_actor_id)?;
+
                             let downstream_worker = ctx
                                 .worker_nodes
                                 .get(&downstream_worker_id)
@@ -1199,7 +1236,7 @@ impl ScaleController {
                         }
                     }
 
-                    let worker = ctx.parallel_unit_id_to_worker(new_parallel_unit_id)?;
+                    let worker = ctx.worker_nodes.get(worker_id).unwrap();
 
                     node_actors_to_create
                         .entry(worker.id)
@@ -1273,14 +1310,10 @@ impl ScaleController {
                 .fragment_type_mask;
             let injectable = TableFragments::is_injectable(fragment_type_mask);
 
-            if let Some(actor_pu_maps) = fragment_actors_to_create.get(&fragment_id).cloned() {
-                for (actor_id, parallel_unit_id) in actor_pu_maps {
-                    let worker_id = ctx
-                        .parallel_unit_id_to_worker_id
-                        .get(&parallel_unit_id)
-                        .with_context(|| format!("parallel unit {} not found", parallel_unit_id))?;
+            if let Some(actor_worker_maps) = fragment_actors_to_create.get(&fragment_id).cloned() {
+                for (actor_id, worker_id) in actor_worker_maps {
                     actors_to_create
-                        .entry(*worker_id)
+                        .entry(worker_id)
                         .or_default()
                         .push(actor_id);
                 }
@@ -1296,14 +1329,7 @@ impl ScaleController {
             let actors_after_reschedule =
                 fragment_actors_after_reschedule.get(&fragment_id).unwrap();
 
-            let parallel_unit_to_actor_after_reschedule: BTreeMap<_, _> = actors_after_reschedule
-                .iter()
-                .map(|(actor_id, parallel_unit_id)| {
-                    (*parallel_unit_id as ParallelUnitId, *actor_id as ActorId)
-                })
-                .collect();
-
-            assert!(!parallel_unit_to_actor_after_reschedule.is_empty());
+            assert!(!actors_after_reschedule.is_empty());
 
             let fragment = ctx.fragment_map.get(&fragment_id).unwrap();
 
@@ -1323,11 +1349,8 @@ impl ScaleController {
                 FragmentDistributionType::Hash => {
                     if !in_degree_types.contains(&DispatcherType::Hash) {
                         None
-                    } else if parallel_unit_to_actor_after_reschedule.len() == 1 {
-                        let actor_id = parallel_unit_to_actor_after_reschedule
-                            .into_values()
-                            .next()
-                            .unwrap();
+                    } else if actors_after_reschedule.len() == 1 {
+                        let actor_id = actors_after_reschedule.keys().next().cloned().unwrap();
                         Some(ActorMapping::new_single(actor_id))
                     } else {
                         // Changes of the bitmap must occur in the case of HashDistribution
@@ -1434,22 +1457,14 @@ impl ScaleController {
         let mut fragment_created_actors = HashMap::new();
         for (fragment_id, actors_to_create) in &fragment_actors_to_create {
             let mut created_actors = HashMap::new();
-            for (actor_id, parallel_unit_id) in actors_to_create {
+            for (actor_id, worker_id) in actors_to_create {
                 let actor = new_created_actors.get(actor_id).cloned().unwrap();
-                let worker_id = ctx
-                    .parallel_unit_id_to_worker_id
-                    .get(parallel_unit_id)
-                    .with_context(|| format!("parallel unit {} not found", parallel_unit_id))?;
-
                 created_actors.insert(
                     *actor_id,
                     (
                         actor,
                         ActorStatus {
-                            parallel_unit: Some(ParallelUnit {
-                                id: *parallel_unit_id,
-                                worker_node_id: *worker_id,
-                            }),
+                            location: PbActorLocation::from_worker(*worker_id),
                             state: ActorState::Inactive as i32,
                         },
                     ),
@@ -1474,62 +1489,72 @@ impl ScaleController {
 
     async fn arrange_reschedules(
         &self,
-        reschedule: &HashMap<FragmentId, ParallelUnitReschedule>,
+        reschedule: &HashMap<FragmentId, WorkerReschedule>,
         ctx: &RescheduleContext,
     ) -> MetaResult<(
-        HashMap<FragmentId, BTreeMap<ActorId, ParallelUnitId>>,
-        HashMap<FragmentId, BTreeMap<ActorId, ParallelUnitId>>,
+        HashMap<FragmentId, BTreeMap<ActorId, WorkerId>>,
+        HashMap<FragmentId, BTreeMap<ActorId, WorkerId>>,
     )> {
         let mut fragment_actors_to_remove = HashMap::with_capacity(reschedule.len());
         let mut fragment_actors_to_create = HashMap::with_capacity(reschedule.len());
 
-        for (
-            fragment_id,
-            ParallelUnitReschedule {
-                added_parallel_units,
-                removed_parallel_units,
-            },
-        ) in reschedule
-        {
+        for (fragment_id, WorkerReschedule { worker_actor_diff }) in reschedule {
             let fragment = ctx.fragment_map.get(fragment_id).unwrap();
 
-            // Actor Id => Parallel Unit Id
+            // Actor Id => Worker Id
             let mut actors_to_remove = BTreeMap::new();
             let mut actors_to_create = BTreeMap::new();
 
-            let parallel_unit_to_actor: HashMap<_, _> = fragment
-                .actors
+            // NOTE(important): The value needs to be a BTreeSet to ensure that the actors on the worker are sorted in ascending order.
+            let mut worker_to_actors = HashMap::new();
+
+            for actor in &fragment.actors {
+                let worker_id = ctx.actor_id_to_worker_id(&actor.actor_id).unwrap();
+                worker_to_actors
+                    .entry(worker_id)
+                    .or_insert(BTreeSet::new())
+                    .insert(actor.actor_id as ActorId);
+            }
+
+            let decreased_actor_count = worker_actor_diff
                 .iter()
-                .map(|actor| {
-                    ctx.actor_id_to_parallel_unit(&actor.actor_id)
-                        .map(|parallel_unit| {
-                            (
-                                parallel_unit.id as ParallelUnitId,
-                                actor.actor_id as ActorId,
-                            )
-                        })
-                })
-                .try_collect()?;
+                .filter(|(_, change)| change.is_negative())
+                .map(|(worker_id, change)| (worker_id, change.unsigned_abs()));
 
-            for removed_parallel_unit_id in removed_parallel_units {
-                if let Some(removed_actor_id) = parallel_unit_to_actor.get(removed_parallel_unit_id)
-                {
-                    actors_to_remove.insert(*removed_actor_id, *removed_parallel_unit_id);
+            for (worker_id, n) in decreased_actor_count {
+                if let Some(actor_ids) = worker_to_actors.get(worker_id) {
+                    assert!(actor_ids.len() >= n);
+
+                    let removed_actors: Vec<_> = actor_ids
+                        .iter()
+                        .skip(actor_ids.len().saturating_sub(n))
+                        .cloned()
+                        .collect();
+
+                    for actor in removed_actors {
+                        actors_to_remove.insert(actor, *worker_id);
+                    }
                 }
             }
 
-            for created_parallel_unit_id in added_parallel_units {
-                let id = match self.env.id_gen_manager() {
-                    IdGenManagerImpl::Kv(mgr) => {
-                        mgr.generate::<{ IdCategory::Actor }>().await? as ActorId
-                    }
-                    IdGenManagerImpl::Sql(mgr) => {
-                        let id = mgr.generate_interval::<{ IdCategory::Actor }>(1);
-                        id as ActorId
-                    }
-                };
+            let increased_actor_count = worker_actor_diff
+                .iter()
+                .filter(|(_, change)| change.is_positive());
+
+            for (worker, n) in increased_actor_count {
+                for _ in 0..*n {
+                    let id = match self.env.id_gen_manager() {
+                        IdGenManagerImpl::Kv(mgr) => {
+                            mgr.generate::<{ IdCategory::Actor }>().await? as ActorId
+                        }
+                        IdGenManagerImpl::Sql(mgr) => {
+                            let id = mgr.generate_interval::<{ IdCategory::Actor }>(1);
+                            id as ActorId
+                        }
+                    };
 
-                actors_to_create.insert(id, *created_parallel_unit_id);
+                    actors_to_create.insert(id, *worker);
+                }
             }
 
             if !actors_to_remove.is_empty() {
@@ -1541,6 +1566,24 @@ impl ScaleController {
             }
         }
 
+        // sanity checking
+        for actors_to_remove in fragment_actors_to_remove.values() {
+            for actor_id in actors_to_remove.keys() {
+                let actor = ctx.actor_map.get(actor_id).unwrap();
+                for dispatcher in &actor.dispatcher {
+                    if DispatcherType::NoShuffle == dispatcher.get_type().unwrap() {
+                        let downstream_actor_id = dispatcher.downstream_actor_id.iter().exactly_one().expect("there should be only one downstream actor id in NO_SHUFFLE dispatcher");
+
+                        let _should_exists = fragment_actors_to_remove
+                            .get(&(dispatcher.dispatcher_id as FragmentId))
+                            .expect("downstream fragment of NO_SHUFFLE relation should be in the removing map")
+                            .get(downstream_actor_id)
+                            .expect("downstream actor of NO_SHUFFLE relation should be in the removing map");
+                    }
+                }
+            }
+        }
+
         Ok((fragment_actors_to_remove, fragment_actors_to_create))
     }
 
@@ -1548,8 +1591,8 @@ impl ScaleController {
     /// overall changes, and is used to handle cascading updates
     fn modify_actor_upstream_and_downstream(
         ctx: &RescheduleContext,
-        fragment_actors_to_remove: &HashMap<FragmentId, BTreeMap<ActorId, ParallelUnitId>>,
-        fragment_actors_to_create: &HashMap<FragmentId, BTreeMap<ActorId, ParallelUnitId>>,
+        fragment_actors_to_remove: &HashMap<FragmentId, BTreeMap<ActorId, WorkerId>>,
+        fragment_actors_to_create: &HashMap<FragmentId, BTreeMap<ActorId, WorkerId>>,
         fragment_actor_bitmap: &HashMap<FragmentId, HashMap<ActorId, Bitmap>>,
         no_shuffle_upstream_actor_map: &HashMap<ActorId, HashMap<FragmentId, ActorId>>,
         no_shuffle_downstream_actors_map: &HashMap<ActorId, HashMap<FragmentId, ActorId>>,
@@ -1713,8 +1756,9 @@ impl ScaleController {
                 .metadata_manager
                 .running_fragment_parallelisms(Some(reschedules.keys().cloned().collect()))
                 .await?;
-            let serving_vnode_mapping = Arc::new(ServingVnodeMapping::default());
-            let (upserted, failed) = serving_vnode_mapping.upsert(streaming_parallelisms, &workers);
+            let serving_worker_slot_mapping = Arc::new(ServingVnodeMapping::default());
+            let (upserted, failed) =
+                serving_worker_slot_mapping.upsert(streaming_parallelisms, &workers);
             if !upserted.is_empty() {
                 tracing::debug!(
                     "Update serving vnode mapping for fragments {:?}.",
@@ -1771,7 +1815,7 @@ impl ScaleController {
     }
 
     // FIXME: should be removed
-    async fn list_all_table_fragments(&self) -> MetaResult<Vec<model::TableFragments>> {
+    pub(crate) async fn list_all_table_fragments(&self) -> MetaResult<Vec<model::TableFragments>> {
         use crate::model::MetadataModel;
         let all_table_fragments = match &self.metadata_manager {
             MetadataManager::V1(mgr) => mgr.fragment_manager.list_table_fragments().await,
@@ -1790,7 +1834,7 @@ impl ScaleController {
     pub async fn generate_table_resize_plan(
         &self,
         policy: TableResizePolicy,
-    ) -> MetaResult<HashMap<FragmentId, ParallelUnitReschedule>> {
+    ) -> MetaResult<HashMap<FragmentId, WorkerReschedule>> {
         let TableResizePolicy {
             worker_ids,
             table_parallelisms,
@@ -1814,18 +1858,14 @@ impl ScaleController {
             .filter(|worker| worker_ids.contains(&worker.id))
             .collect::<Vec<_>>();
 
-        let worker_parallel_units = workers
-            .iter()
-            .map(|worker| {
-                (
-                    worker.id,
-                    worker
-                        .parallel_units
-                        .iter()
-                        .map(|parallel_unit| parallel_unit.id as ParallelUnitId)
-                        .collect::<BTreeSet<_>>(),
-                )
-            })
+        let workers: HashMap<_, _> = workers
+            .into_iter()
+            .map(|worker| (worker.id, worker))
+            .collect();
+
+        let worker_slots = workers
+            .values()
+            .map(|worker| (worker.id, worker.parallelism as usize))
             .collect::<BTreeMap<_, _>>();
 
         // index for no shuffle relation
@@ -1834,8 +1874,8 @@ impl ScaleController {
 
         // index for fragment_id -> distribution_type
         let mut fragment_distribution_map = HashMap::new();
-        // index for actor -> parallel_unit
-        let mut actor_status = HashMap::new();
+        // index for actor -> worker id
+        let mut actor_location = HashMap::new();
         // index for table_id -> [fragment_id]
         let mut table_fragment_id_map = HashMap::new();
         // index for fragment_id -> [actor_id]
@@ -1846,7 +1886,7 @@ impl ScaleController {
             no_shuffle_source_fragment_ids: &mut HashSet<FragmentId>,
             no_shuffle_target_fragment_ids: &mut HashSet<FragmentId>,
             fragment_distribution_map: &mut HashMap<FragmentId, FragmentDistributionType>,
-            actor_status: &mut HashMap<ActorId, ActorStatus>,
+            actor_location: &mut HashMap<ActorId, WorkerId>,
             table_fragment_id_map: &mut HashMap<u32, HashSet<FragmentId>>,
             fragment_actor_id_map: &mut HashMap<FragmentId, HashSet<u32>>,
             table_fragments: &BTreeMap<TableId, TableFragments>,
@@ -1914,7 +1954,9 @@ impl ScaleController {
                         .insert(*fragment_id);
                 }
 
-                actor_status.extend(table_fragments.actor_status.clone());
+                for (actor_id, status) in &table_fragments.actor_status {
+                    actor_location.insert(*actor_id, status.worker_id());
+                }
             }
 
             Ok(())
@@ -1927,7 +1969,7 @@ impl ScaleController {
                     &mut no_shuffle_source_fragment_ids,
                     &mut no_shuffle_target_fragment_ids,
                     &mut fragment_distribution_map,
-                    &mut actor_status,
+                    &mut actor_location,
                     &mut table_fragment_id_map,
                     &mut fragment_actor_id_map,
                     guard.table_fragments(),
@@ -1944,7 +1986,7 @@ impl ScaleController {
                     &mut no_shuffle_source_fragment_ids,
                     &mut no_shuffle_target_fragment_ids,
                     &mut fragment_distribution_map,
-                    &mut actor_status,
+                    &mut actor_location,
                     &mut table_fragment_id_map,
                     &mut fragment_actor_id_map,
                     &all_table_fragments,
@@ -1963,25 +2005,19 @@ impl ScaleController {
                     continue;
                 }
 
-                let fragment_parallel_unit_ids: BTreeSet<ParallelUnitId> = fragment_actor_id_map
-                    .get(&fragment_id)
-                    .unwrap()
-                    .iter()
-                    .map(|actor_id| {
-                        actor_status
-                            .get(actor_id)
-                            .and_then(|status| status.parallel_unit.clone())
-                            .unwrap()
-                            .id as ParallelUnitId
-                    })
-                    .collect();
+                let mut fragment_slots: BTreeMap<WorkerId, usize> = BTreeMap::new();
+
+                for actor_id in fragment_actor_id_map.get(&fragment_id).unwrap() {
+                    let worker_id = actor_location.get(actor_id).unwrap();
+
+                    *fragment_slots.entry(*worker_id).or_default() += 1;
+                }
 
-                let all_available_parallel_unit_ids: BTreeSet<_> =
-                    worker_parallel_units.values().flatten().cloned().collect();
+                let all_available_slots: usize = worker_slots.values().cloned().sum();
 
-                if all_available_parallel_unit_ids.is_empty() {
+                if all_available_slots == 0 {
                     bail!(
-                        "No schedulable ParallelUnits available for fragment {}",
+                        "No schedulable slots available for fragment {}",
                         fragment_id
                     );
                 }
@@ -1989,31 +2025,35 @@ impl ScaleController {
                 match fragment_distribution_map.get(&fragment_id).unwrap() {
                     FragmentDistributionType::Unspecified => unreachable!(),
                     FragmentDistributionType::Single => {
-                        let single_parallel_unit_id =
-                            fragment_parallel_unit_ids.iter().exactly_one().unwrap();
+                        let (single_worker_id, should_be_one) =
+                            fragment_slots.iter().exactly_one().unwrap();
+
+                        assert_eq!(*should_be_one, 1);
 
-                        if all_available_parallel_unit_ids.contains(single_parallel_unit_id) {
+                        if worker_slots.contains_key(single_worker_id) {
                             // NOTE: shall we continue?
                             continue;
                         }
 
-                        let units = schedule_units_for_slots(&worker_parallel_units, 1, table_id)?;
+                        let units = schedule_units_for_slots(&worker_slots, 1, table_id)?;
 
-                        let chosen_target_parallel_unit_id = units
-                            .values()
-                            .flatten()
-                            .cloned()
-                            .exactly_one()
-                            .ok()
-                            .with_context(|| format!("Cannot find a single target ParallelUnit for fragment {fragment_id}"))?;
+                        let (chosen_target_worker_id, should_be_one) =
+                            units.iter().exactly_one().ok().with_context(|| {
+                                format!(
+                                    "Cannot find a single target worker for fragment {fragment_id}"
+                                )
+                            })?;
+
+                        assert_eq!(*should_be_one, 1);
+                        assert_ne!(*chosen_target_worker_id, *single_worker_id);
 
                         target_plan.insert(
                             fragment_id,
-                            ParallelUnitReschedule {
-                                added_parallel_units: BTreeSet::from([
-                                    chosen_target_parallel_unit_id,
+                            WorkerReschedule {
+                                worker_actor_diff: BTreeMap::from_iter(vec![
+                                    (*chosen_target_worker_id, 1),
+                                    (*single_worker_id, -1),
                                 ]),
-                                removed_parallel_units: BTreeSet::from([*single_parallel_unit_id]),
                             },
                         );
                     }
@@ -2021,38 +2061,18 @@ impl ScaleController {
                         TableParallelism::Adaptive => {
                             target_plan.insert(
                                 fragment_id,
-                                Self::diff_parallel_unit_change(
-                                    &fragment_parallel_unit_ids,
-                                    &all_available_parallel_unit_ids,
-                                ),
+                                Self::diff_worker_slot_changes(&fragment_slots, &worker_slots),
                             );
                         }
-                        TableParallelism::Fixed(mut n) => {
-                            let available_parallelism = all_available_parallel_unit_ids.len();
-
-                            if n > available_parallelism {
-                                warn!(
-                                    "not enough parallel units available for job {} fragment {}, required {}, resetting to {}",
-                                    table_id,
-                                    fragment_id,
-                                    n,
-                                    available_parallelism,
-                                );
-
-                                n = available_parallelism;
-                            }
-
-                            let rebalance_result =
-                                schedule_units_for_slots(&worker_parallel_units, n, table_id)?;
-
-                            let target_parallel_unit_ids =
-                                rebalance_result.into_values().flatten().collect();
+                        TableParallelism::Fixed(n) => {
+                            let target_worker_slots =
+                                schedule_units_for_slots(&worker_slots, n, table_id)?;
 
                             target_plan.insert(
                                 fragment_id,
-                                Self::diff_parallel_unit_change(
-                                    &fragment_parallel_unit_ids,
-                                    &target_parallel_unit_ids,
+                                Self::diff_worker_slot_changes(
+                                    &fragment_slots,
+                                    &target_worker_slots,
                                 ),
                             );
                         }
@@ -2064,359 +2084,12 @@ impl ScaleController {
             }
         }
 
-        target_plan.retain(|_, plan| {
-            !(plan.added_parallel_units.is_empty() && plan.removed_parallel_units.is_empty())
-        });
-
-        Ok(target_plan)
-    }
-
-    pub async fn generate_stable_resize_plan(
-        &self,
-        policy: StableResizePolicy,
-        parallel_unit_hints: Option<HashMap<WorkerId, HashSet<ParallelUnitId>>>,
-    ) -> MetaResult<HashMap<FragmentId, ParallelUnitReschedule>> {
-        let StableResizePolicy {
-            fragment_worker_changes,
-        } = policy;
-
-        let mut target_plan = HashMap::with_capacity(fragment_worker_changes.len());
-
-        let workers = self
-            .metadata_manager
-            .list_active_streaming_compute_nodes()
-            .await?;
-
-        let unschedulable_worker_ids = Self::filter_unschedulable_workers(&workers);
-
-        for changes in fragment_worker_changes.values() {
-            for worker_id in &changes.include_worker_ids {
-                if unschedulable_worker_ids.contains(worker_id) {
-                    bail!("Cannot include unscheduable worker {}", worker_id)
-                }
-            }
-        }
-
-        let worker_parallel_units = workers
-            .iter()
-            .map(|worker| {
-                (
-                    worker.id,
-                    worker
-                        .parallel_units
-                        .iter()
-                        .map(|parallel_unit| parallel_unit.id as ParallelUnitId)
-                        .collect::<HashSet<_>>(),
-                )
-            })
-            .collect::<HashMap<_, _>>();
-
-        // FIXME: only need actor id and dispatcher info, avoid clone it.
-        let mut actor_map = HashMap::new();
-        let mut actor_status = HashMap::new();
-        // FIXME: only need fragment distribution info, should avoid clone it.
-        let mut fragment_map = HashMap::new();
-        let mut fragment_parallelism = HashMap::new();
-
-        // We are reusing code for the metadata manager of both V1 and V2, which will be deprecated in the future.
-        fn fulfill_index_by_table_fragments_ref(
-            actor_map: &mut HashMap<u32, CustomActorInfo>,
-            actor_status: &mut HashMap<ActorId, ActorStatus>,
-            fragment_map: &mut HashMap<FragmentId, CustomFragmentInfo>,
-            fragment_parallelism: &mut HashMap<FragmentId, TableParallelism>,
-            table_fragments: &TableFragments,
-        ) {
-            for (fragment_id, fragment) in &table_fragments.fragments {
-                for actor in &fragment.actors {
-                    actor_map.insert(actor.actor_id, CustomActorInfo::from(actor));
-                }
-
-                fragment_map.insert(*fragment_id, CustomFragmentInfo::from(fragment));
-
-                fragment_parallelism.insert(*fragment_id, table_fragments.assigned_parallelism);
-            }
-
-            actor_status.extend(table_fragments.actor_status.clone());
-        }
-
-        match &self.metadata_manager {
-            MetadataManager::V1(mgr) => {
-                let guard = mgr.fragment_manager.get_fragment_read_guard().await;
-
-                for table_fragments in guard.table_fragments().values() {
-                    fulfill_index_by_table_fragments_ref(
-                        &mut actor_map,
-                        &mut actor_status,
-                        &mut fragment_map,
-                        &mut fragment_parallelism,
-                        table_fragments,
-                    );
-                }
-            }
-            MetadataManager::V2(_) => {
-                let all_table_fragments = self.list_all_table_fragments().await?;
-
-                for table_fragments in &all_table_fragments {
-                    fulfill_index_by_table_fragments_ref(
-                        &mut actor_map,
-                        &mut actor_status,
-                        &mut fragment_map,
-                        &mut fragment_parallelism,
-                        table_fragments,
-                    );
-                }
-            }
-        };
-
-        let mut no_shuffle_source_fragment_ids = HashSet::new();
-        let mut no_shuffle_target_fragment_ids = HashSet::new();
-
-        Self::build_no_shuffle_relation_index(
-            &actor_map,
-            &mut no_shuffle_source_fragment_ids,
-            &mut no_shuffle_target_fragment_ids,
-        );
-
-        let mut fragment_dispatcher_map = HashMap::new();
-        Self::build_fragment_dispatcher_index(&actor_map, &mut fragment_dispatcher_map);
-
-        #[derive(PartialEq, Eq, Clone)]
-        struct WorkerChanges {
-            include_worker_ids: BTreeSet<WorkerId>,
-            exclude_worker_ids: BTreeSet<WorkerId>,
-            target_parallelism: Option<usize>,
-            target_parallelism_per_worker: Option<usize>,
-        }
-
-        let mut fragment_worker_changes: HashMap<_, _> = fragment_worker_changes
-            .into_iter()
-            .map(|(fragment_id, changes)| {
-                (
-                    fragment_id as FragmentId,
-                    WorkerChanges {
-                        include_worker_ids: changes.include_worker_ids.into_iter().collect(),
-                        exclude_worker_ids: changes.exclude_worker_ids.into_iter().collect(),
-                        target_parallelism: changes.target_parallelism.map(|p| p as usize),
-                        target_parallelism_per_worker: changes
-                            .target_parallelism_per_worker
-                            .map(|p| p as usize),
-                    },
-                )
-            })
-            .collect();
-
-        Self::resolve_no_shuffle_upstream_fragments(
-            &mut fragment_worker_changes,
-            &fragment_map,
-            &no_shuffle_source_fragment_ids,
-            &no_shuffle_target_fragment_ids,
-        )?;
-
-        for (
-            fragment_id,
-            WorkerChanges {
-                include_worker_ids,
-                exclude_worker_ids,
-                target_parallelism,
-                target_parallelism_per_worker,
-            },
-        ) in fragment_worker_changes
-        {
-            let fragment = match fragment_map.get(&fragment_id) {
-                None => bail!("Fragment id {} not found", fragment_id),
-                Some(fragment) => fragment,
-            };
-
-            let intersection_ids = include_worker_ids
-                .intersection(&exclude_worker_ids)
-                .collect_vec();
-
-            if !intersection_ids.is_empty() {
-                bail!(
-                    "Include worker ids {:?} and exclude worker ids {:?} have intersection {:?}",
-                    include_worker_ids,
-                    exclude_worker_ids,
-                    intersection_ids
-                );
-            }
-
-            for worker_id in include_worker_ids.iter().chain(exclude_worker_ids.iter()) {
-                if !worker_parallel_units.contains_key(worker_id)
-                    && !parallel_unit_hints
-                        .as_ref()
-                        .map(|hints| hints.contains_key(worker_id))
-                        .unwrap_or(false)
-                {
-                    bail!("Worker id {} not found", worker_id);
-                }
-            }
-
-            let fragment_parallel_unit_ids: BTreeSet<_> = fragment
-                .actors
-                .iter()
-                .map(|actor| {
-                    actor_status
-                        .get(&actor.actor_id)
-                        .and_then(|status| status.parallel_unit.clone())
-                        .unwrap()
-                        .id as ParallelUnitId
-                })
-                .collect();
-
-            let worker_to_parallel_unit_ids = |worker_ids: &BTreeSet<WorkerId>| {
-                worker_ids
-                    .iter()
-                    .flat_map(|worker_id| {
-                        worker_parallel_units
-                            .get(worker_id)
-                            .or_else(|| {
-                                parallel_unit_hints
-                                    .as_ref()
-                                    .and_then(|hints| hints.get(worker_id))
-                            })
-                            .expect("worker id should be valid")
-                    })
-                    .cloned()
-                    .collect_vec()
-            };
-
-            let include_worker_parallel_unit_ids = worker_to_parallel_unit_ids(&include_worker_ids);
-            let exclude_worker_parallel_unit_ids = worker_to_parallel_unit_ids(&exclude_worker_ids);
-
-            fn refilter_parallel_unit_id_by_target_parallelism(
-                worker_parallel_units: &HashMap<u32, HashSet<ParallelUnitId>>,
-                include_worker_ids: &BTreeSet<WorkerId>,
-                include_worker_parallel_unit_ids: &[ParallelUnitId],
-                target_parallel_unit_ids: &mut BTreeSet<ParallelUnitId>,
-                target_parallelism_per_worker: usize,
-            ) {
-                let limited_worker_parallel_unit_ids = include_worker_ids
-                    .iter()
-                    .flat_map(|worker_id| {
-                        worker_parallel_units
-                            .get(worker_id)
-                            .cloned()
-                            .unwrap()
-                            .into_iter()
-                            .sorted()
-                            .take(target_parallelism_per_worker)
-                    })
-                    .collect_vec();
-
-                // remove all the parallel units in the limited workers
-                target_parallel_unit_ids
-                    .retain(|id| !include_worker_parallel_unit_ids.contains(id));
-
-                // then we re-add the limited parallel units from the limited workers
-                target_parallel_unit_ids.extend(limited_worker_parallel_unit_ids.into_iter());
-            }
-            match fragment.distribution_type() {
-                FragmentDistributionType::Unspecified => unreachable!(),
-                FragmentDistributionType::Single => {
-                    let single_parallel_unit_id =
-                        fragment_parallel_unit_ids.iter().exactly_one().unwrap();
-
-                    let mut target_parallel_unit_ids: BTreeSet<_> = worker_parallel_units
-                        .keys()
-                        .filter(|id| !unschedulable_worker_ids.contains(*id))
-                        .filter(|id| !exclude_worker_ids.contains(*id))
-                        .flat_map(|id| worker_parallel_units.get(id).cloned().unwrap())
-                        .collect();
-
-                    if let Some(target_parallelism_per_worker) = target_parallelism_per_worker {
-                        refilter_parallel_unit_id_by_target_parallelism(
-                            &worker_parallel_units,
-                            &include_worker_ids,
-                            &include_worker_parallel_unit_ids,
-                            &mut target_parallel_unit_ids,
-                            target_parallelism_per_worker,
-                        );
-                    }
-
-                    if target_parallel_unit_ids.is_empty() {
-                        bail!(
-                            "No schedulable ParallelUnits available for single distribution fragment {}",
-                            fragment_id
-                        );
-                    }
-
-                    if !target_parallel_unit_ids.contains(single_parallel_unit_id) {
-                        let sorted_target_parallel_unit_ids =
-                            target_parallel_unit_ids.into_iter().sorted().collect_vec();
-
-                        let chosen_target_parallel_unit_id = sorted_target_parallel_unit_ids
-                            [fragment_id as usize % sorted_target_parallel_unit_ids.len()];
-
-                        target_plan.insert(
-                            fragment_id,
-                            ParallelUnitReschedule {
-                                added_parallel_units: BTreeSet::from([
-                                    chosen_target_parallel_unit_id,
-                                ]),
-                                removed_parallel_units: BTreeSet::from([*single_parallel_unit_id]),
-                            },
-                        );
-                    }
-                }
-                FragmentDistributionType::Hash => {
-                    let mut target_parallel_unit_ids: BTreeSet<_> =
-                        fragment_parallel_unit_ids.clone();
-                    target_parallel_unit_ids.extend(include_worker_parallel_unit_ids.iter());
-                    target_parallel_unit_ids
-                        .retain(|id| !exclude_worker_parallel_unit_ids.contains(id));
-
-                    if target_parallel_unit_ids.is_empty() {
-                        bail!(
-                            "No schedulable ParallelUnits available for fragment {}",
-                            fragment_id
-                        );
-                    }
-
-                    match (target_parallelism, target_parallelism_per_worker) {
-                        (Some(_), Some(_)) => {
-                            bail!("Cannot specify both target parallelism and target parallelism per worker");
-                        }
-                        (Some(target_parallelism), _) => {
-                            if target_parallel_unit_ids.len() < target_parallelism {
-                                bail!("Target parallelism {} is greater than schedulable ParallelUnits {}", target_parallelism, target_parallel_unit_ids.len());
-                            }
-
-                            target_parallel_unit_ids = target_parallel_unit_ids
-                                .into_iter()
-                                .take(target_parallelism)
-                                .collect();
-                        }
-                        (_, Some(target_parallelism_per_worker)) => {
-                            refilter_parallel_unit_id_by_target_parallelism(
-                                &worker_parallel_units,
-                                &include_worker_ids,
-                                &include_worker_parallel_unit_ids,
-                                &mut target_parallel_unit_ids,
-                                target_parallelism_per_worker,
-                            );
-                        }
-                        _ => {}
-                    }
-
-                    target_plan.insert(
-                        fragment_id,
-                        Self::diff_parallel_unit_change(
-                            &fragment_parallel_unit_ids,
-                            &target_parallel_unit_ids,
-                        ),
-                    );
-                }
-            }
-        }
-
-        target_plan.retain(|_, plan| {
-            !(plan.added_parallel_units.is_empty() && plan.removed_parallel_units.is_empty())
-        });
+        target_plan.retain(|_, plan| !plan.worker_actor_diff.is_empty());
 
         Ok(target_plan)
     }
 
-    fn filter_unschedulable_workers(workers: &[WorkerNode]) -> HashSet<WorkerId> {
+    pub(crate) fn filter_unschedulable_workers(workers: &[WorkerNode]) -> HashSet<WorkerId> {
         workers
             .iter()
             .filter(|worker| {
@@ -2430,38 +2103,51 @@ impl ScaleController {
             .collect()
     }
 
-    fn diff_parallel_unit_change(
-        fragment_parallel_unit_ids: &BTreeSet<ParallelUnitId>,
-        target_parallel_unit_ids: &BTreeSet<ParallelUnitId>,
-    ) -> ParallelUnitReschedule {
-        let to_expand_parallel_units = target_parallel_unit_ids
-            .difference(fragment_parallel_unit_ids)
-            .cloned()
-            .collect();
+    fn diff_worker_slot_changes(
+        fragment_worker_slots: &BTreeMap<WorkerId, usize>,
+        target_worker_slots: &BTreeMap<WorkerId, usize>,
+    ) -> WorkerReschedule {
+        let mut increased_actor_count: BTreeMap<WorkerId, usize> = BTreeMap::new();
+        let mut decreased_actor_count: BTreeMap<WorkerId, usize> = BTreeMap::new();
 
-        let to_shrink_parallel_units = fragment_parallel_unit_ids
-            .difference(target_parallel_unit_ids)
-            .cloned()
-            .collect();
+        for (&worker_id, &target_slots) in target_worker_slots {
+            let &current_slots = fragment_worker_slots.get(&worker_id).unwrap_or(&0);
 
-        ParallelUnitReschedule {
-            added_parallel_units: to_expand_parallel_units,
-            removed_parallel_units: to_shrink_parallel_units,
+            if target_slots > current_slots {
+                increased_actor_count.insert(worker_id, target_slots - current_slots);
+            }
         }
-    }
 
-    pub async fn get_reschedule_plan(
-        &self,
-        policy: Policy,
-    ) -> MetaResult<HashMap<FragmentId, ParallelUnitReschedule>> {
-        match policy {
-            Policy::StableResizePolicy(resize) => {
-                self.generate_stable_resize_plan(resize, None).await
+        for (&worker_id, &current_slots) in fragment_worker_slots {
+            let &target_slots = target_worker_slots.get(&worker_id).unwrap_or(&0);
+
+            if current_slots > target_slots {
+                decreased_actor_count.insert(worker_id, current_slots - target_slots);
             }
         }
+
+        let worker_ids: HashSet<_> = increased_actor_count
+            .keys()
+            .chain(decreased_actor_count.keys())
+            .cloned()
+            .collect();
+
+        let mut worker_actor_diff = BTreeMap::new();
+
+        for worker_id in worker_ids {
+            let increased = increased_actor_count.remove(&worker_id).unwrap_or(0) as isize;
+            let decreased = decreased_actor_count.remove(&worker_id).unwrap_or(0) as isize;
+            let change = increased - decreased;
+
+            assert_ne!(change, 0);
+
+            worker_actor_diff.insert(worker_id, change);
+        }
+
+        WorkerReschedule { worker_actor_diff }
     }
 
-    pub fn build_no_shuffle_relation_index(
+    fn build_no_shuffle_relation_index(
         actor_map: &HashMap<ActorId, CustomActorInfo>,
         no_shuffle_source_fragment_ids: &mut HashSet<FragmentId>,
         no_shuffle_target_fragment_ids: &mut HashSet<FragmentId>,
@@ -2489,7 +2175,7 @@ impl ScaleController {
         }
     }
 
-    pub fn build_fragment_dispatcher_index(
+    fn build_fragment_dispatcher_index(
         actor_map: &HashMap<ActorId, CustomActorInfo>,
         fragment_dispatcher_map: &mut HashMap<FragmentId, HashMap<FragmentId, DispatcherType>>,
     ) {
@@ -2628,8 +2314,8 @@ impl ScaleController {
     }
 }
 
-// At present, for table level scaling, we use the strategy TableResizePolicy.
-// Currently, this is used as an internal interface, so it won’t be included in Protobuf for the time being.
+/// At present, for table level scaling, we use the strategy `TableResizePolicy`.
+/// Currently, this is used as an internal interface, so it won’t be included in Protobuf.
 pub struct TableResizePolicy {
     pub(crate) worker_ids: BTreeSet<WorkerId>,
     pub(crate) table_parallelisms: HashMap<u32, TableParallelism>,
@@ -2644,9 +2330,16 @@ impl GlobalStreamManager {
         self.scale_controller.reschedule_lock.write().await
     }
 
+    /// The entrypoint of rescheduling actors.
+    ///
+    /// Used by:
+    /// - The directly exposed low-level API `risingwave_meta_service::scale_service::ScaleService` (`risectl meta reschedule`)
+    /// - High-level parallelism control API
+    ///     * manual `ALTER [TABLE | INDEX | MATERIALIZED VIEW | SINK] SET PARALLELISM`
+    ///     * automatic parallelism control for [`TableParallelism::Adaptive`] when worker nodes changed
     pub async fn reschedule_actors(
         &self,
-        reschedules: HashMap<FragmentId, ParallelUnitReschedule>,
+        reschedules: HashMap<FragmentId, WorkerReschedule>,
         options: RescheduleOptions,
         table_parallelism: Option<HashMap<TableId, TableParallelism>>,
     ) -> MetaResult<()> {
@@ -2667,7 +2360,7 @@ impl GlobalStreamManager {
     async fn reschedule_actors_impl(
         &self,
         revert_funcs: &mut Vec<BoxFuture<'_, ()>>,
-        reschedules: HashMap<FragmentId, ParallelUnitReschedule>,
+        reschedules: HashMap<FragmentId, WorkerReschedule>,
         options: RescheduleOptions,
         table_parallelism: Option<HashMap<TableId, TableParallelism>>,
     ) -> MetaResult<()> {
@@ -2675,7 +2368,7 @@ impl GlobalStreamManager {
 
         let (reschedule_fragment, applied_reschedules) = self
             .scale_controller
-            .prepare_reschedule_command(reschedules, options, table_parallelism.as_mut())
+            .analyze_reschedule_plan(reschedules, options, table_parallelism.as_mut())
             .await?;
 
         tracing::debug!("reschedule plan: {:?}", reschedule_fragment);
@@ -2736,6 +2429,14 @@ impl GlobalStreamManager {
         Ok(())
     }
 
+    /// When new worker nodes joined, or the parallelism of existing worker nodes changed,
+    /// examines if there are any jobs can be scaled, and scales them if found.
+    ///
+    /// This method will iterate over all `CREATED` jobs, and can be repeatedly called.
+    ///
+    /// Returns
+    /// - `Ok(false)` if no jobs can be scaled;
+    /// - `Ok(true)` if some jobs are scaled, and it is possible that there are more jobs can be scaled.
     async fn trigger_parallelism_control(&self) -> MetaResult<bool> {
         let background_streaming_jobs = self
             .metadata_manager
@@ -2857,7 +2558,9 @@ impl GlobalStreamManager {
 
         for batch in batches {
             let parallelisms: HashMap<_, _> = batch.into_iter().collect();
-
+            // `table_parallelisms` contains ALL created jobs.
+            // We rely on `generate_table_resize_plan` to check if there are
+            // any jobs that can be scaled.
             let plan = self
                 .scale_controller
                 .generate_table_resize_plan(TableResizePolicy {
@@ -2894,6 +2597,7 @@ impl GlobalStreamManager {
         Ok(true)
     }
 
+    /// Handles notification of worker node activation and deletion, and triggers parallelism control.
     async fn run(&self, mut shutdown_rx: Receiver<()>) {
         tracing::info!("starting automatic parallelism control monitor");
 
@@ -2963,19 +2667,24 @@ impl GlobalStreamManager {
                 notification = local_notification_rx.recv() => {
                     let notification = notification.expect("local notification channel closed in loop of stream manager");
 
+                    // Only maintain the cache for streaming compute nodes.
+                    let worker_is_streaming_compute = |worker: &WorkerNode| {
+                        worker.get_type() == Ok(WorkerType::ComputeNode)
+                            && worker.property.as_ref().unwrap().is_streaming
+                    };
+
                     match notification {
                         LocalNotification::WorkerNodeActivated(worker) => {
-                            match (worker.get_type(), worker.property.as_ref()) {
-                                (Ok(WorkerType::ComputeNode), Some(prop)) if prop.is_streaming => {
-                                    tracing::info!("worker {} activated notification received", worker.id);
-                                }
-                                _ => continue
+                            if !worker_is_streaming_compute(&worker) {
+                                continue;
                             }
 
+                            tracing::info!(worker = worker.id, "worker activated notification received");
+
                             let prev_worker = worker_cache.insert(worker.id, worker.clone());
 
                             match prev_worker {
-                                Some(prev_worker) if prev_worker.parallel_units != worker.parallel_units  => {
+                                Some(prev_worker) if prev_worker.get_parallelism() != worker.get_parallelism()  => {
                                     tracing::info!(worker = worker.id, "worker parallelism changed");
                                     should_trigger = true;
                                 }
@@ -2990,11 +2699,14 @@ impl GlobalStreamManager {
                         // Since our logic for handling passive scale-in is within the barrier manager,
                         // there’s not much we can do here. All we can do is proactively remove the entries from our cache.
                         LocalNotification::WorkerNodeDeleted(worker) => {
+                            if !worker_is_streaming_compute(&worker) {
+                                continue;
+                            }
+
                             match worker_cache.remove(&worker.id) {
                                 Some(prev_worker) => {
                                     tracing::info!(worker = prev_worker.id, "worker removed from stream manager cache");
                                 }
-
                                 None => {
                                     tracing::warn!(worker = worker.id, "worker not found in stream manager cache, but it was removed");
                                 }
@@ -3021,45 +2733,28 @@ impl GlobalStreamManager {
     }
 }
 
-// We redistribute parallel units (which will be ensembles in the future) through a simple consistent hashing ring.
-// Note that we have added some simple logic here to ensure the consistency of the ratio between each slot,
-// especially when equal division is needed.
 pub fn schedule_units_for_slots(
-    slots: &BTreeMap<WorkerId, BTreeSet<ParallelUnitId>>,
+    slots: &BTreeMap<WorkerId, usize>,
     total_unit_size: usize,
     salt: u32,
-) -> MetaResult<BTreeMap<WorkerId, BTreeSet<ParallelUnitId>>> {
+) -> MetaResult<BTreeMap<WorkerId, usize>> {
     let mut ch = ConsistentHashRing::new(salt);
 
-    for (worker_id, parallel_unit_ids) in slots {
-        ch.add_worker(*worker_id, parallel_unit_ids.len() as u32);
+    for (worker_id, parallelism) in slots {
+        ch.add_worker(*worker_id, *parallelism as u32);
     }
 
     let target_distribution = ch.distribute_tasks(total_unit_size as u32)?;
 
-    Ok(slots
-        .iter()
-        .map(|(worker_id, parallel_unit_ids)| {
-            (
-                *worker_id,
-                parallel_unit_ids
-                    .iter()
-                    .take(
-                        target_distribution
-                            .get(worker_id)
-                            .cloned()
-                            .unwrap_or_default() as usize,
-                    )
-                    .cloned()
-                    .collect::<BTreeSet<_>>(),
-            )
-        })
+    Ok(target_distribution
+        .into_iter()
+        .map(|(worker_id, task_count)| (worker_id as WorkerId, task_count as usize))
         .collect())
 }
 
 pub struct ConsistentHashRing {
     ring: BTreeMap<u64, u32>,
-    capacities: BTreeMap<u32, u32>,
+    weights: BTreeMap<u32, u32>,
     virtual_nodes: u32,
     salt: u32,
 }
@@ -3068,7 +2763,7 @@ impl ConsistentHashRing {
     fn new(salt: u32) -> Self {
         ConsistentHashRing {
             ring: BTreeMap::new(),
-            capacities: BTreeMap::new(),
+            weights: BTreeMap::new(),
             virtual_nodes: 1024,
             salt,
         }
@@ -3081,7 +2776,7 @@ impl ConsistentHashRing {
         hasher.finish()
     }
 
-    fn add_worker(&mut self, id: u32, capacity: u32) {
+    fn add_worker(&mut self, id: u32, weight: u32) {
         let virtual_nodes_count = self.virtual_nodes;
 
         for i in 0..virtual_nodes_count {
@@ -3090,21 +2785,17 @@ impl ConsistentHashRing {
             self.ring.insert(hash, id);
         }
 
-        self.capacities.insert(id, capacity);
+        self.weights.insert(id, weight);
     }
 
     fn distribute_tasks(&self, total_tasks: u32) -> MetaResult<BTreeMap<u32, u32>> {
-        let total_capacity = self.capacities.values().sum::<u32>();
-
-        if total_capacity < total_tasks {
-            bail!("Total tasks exceed the total weight of all workers.");
-        }
+        let total_weight = self.weights.values().sum::<u32>();
 
         let mut soft_limits = HashMap::new();
-        for (worker_id, worker_capacity) in &self.capacities {
+        for (worker_id, worker_capacity) in &self.weights {
             soft_limits.insert(
                 *worker_id,
-                (total_tasks as f64 * (*worker_capacity as f64 / total_capacity as f64)).ceil()
+                (total_tasks as f64 * (*worker_capacity as f64 / total_weight as f64)).ceil()
                     as u32,
             );
         }
@@ -3124,10 +2815,7 @@ impl ConsistentHashRing {
             let ring_range = self.ring.range(task_hash..).chain(self.ring.iter());
 
             for (_, &worker_id) in ring_range {
-                let worker_capacity = self.capacities.get(&worker_id).unwrap();
-                let worker_soft_limit = soft_limits.get(&worker_id).unwrap();
-
-                let task_limit = min(*worker_capacity, *worker_soft_limit);
+                let task_limit = *soft_limits.get(&worker_id).unwrap();
 
                 let worker_task_count = task_distribution.entry(worker_id).or_insert(0);
 
@@ -3207,7 +2895,7 @@ mod tests {
         let total_tasks = 10; // More tasks than the total weight
         let task_distribution = ch.distribute_tasks(total_tasks);
 
-        assert!(task_distribution.is_err());
+        assert!(task_distribution.is_ok());
     }
 
     #[test]
diff --git a/src/meta/src/stream/sink.rs b/src/meta/src/stream/sink.rs
index 6b91c52c85f43..90a496823cc48 100644
--- a/src/meta/src/stream/sink.rs
+++ b/src/meta/src/stream/sink.rs
@@ -22,7 +22,7 @@ use crate::MetaResult;
 
 pub async fn validate_sink(prost_sink_catalog: &PbSink) -> MetaResult<()> {
     let sink_catalog = SinkCatalog::from(prost_sink_catalog);
-    let param = SinkParam::from(sink_catalog);
+    let param = SinkParam::try_from_sink_catalog(sink_catalog)?;
 
     let sink = build_sink(param)?;
 
diff --git a/src/meta/src/stream/source_manager.rs b/src/meta/src/stream/source_manager.rs
index 0fe9d4a961427..b56b9e582f940 100644
--- a/src/meta/src/stream/source_manager.rs
+++ b/src/meta/src/stream/source_manager.rs
@@ -23,12 +23,12 @@ use std::time::Duration;
 use anyhow::Context;
 use risingwave_common::catalog::TableId;
 use risingwave_common::metrics::LabelGuardedIntGauge;
-use risingwave_connector::dispatch_source_prop;
 use risingwave_connector::error::ConnectorResult;
 use risingwave_connector::source::{
     ConnectorProperties, SourceEnumeratorContext, SourceEnumeratorInfo, SourceProperties,
     SplitEnumerator, SplitId, SplitImpl, SplitMetaData,
 };
+use risingwave_connector::{dispatch_source_prop, WithOptionsSecResolved};
 use risingwave_pb::catalog::Source;
 use risingwave_pb::source::{ConnectorSplit, ConnectorSplits};
 use risingwave_pb::stream_plan::Dispatcher;
@@ -81,12 +81,16 @@ struct ConnectorSourceWorker<P: SourceProperties> {
 }
 
 fn extract_prop_from_existing_source(source: &Source) -> ConnectorResult<ConnectorProperties> {
-    let mut properties = ConnectorProperties::extract(source.with_properties.clone(), false)?;
+    let options_with_secret =
+        WithOptionsSecResolved::new(source.with_properties.clone(), source.secret_refs.clone());
+    let mut properties = ConnectorProperties::extract(options_with_secret, false)?;
     properties.init_from_pb_source(source);
     Ok(properties)
 }
 fn extract_prop_from_new_source(source: &Source) -> ConnectorResult<ConnectorProperties> {
-    let mut properties = ConnectorProperties::extract(source.with_properties.clone(), true)?;
+    let options_with_secret =
+        WithOptionsSecResolved::new(source.with_properties.clone(), source.secret_refs.clone());
+    let mut properties = ConnectorProperties::extract(options_with_secret, true)?;
     properties.init_from_pb_source(source);
     Ok(properties)
 }
diff --git a/src/meta/src/stream/stream_graph/actor.rs b/src/meta/src/stream/stream_graph/actor.rs
index 7b02920ac3c4d..8847ff1e8c028 100644
--- a/src/meta/src/stream/stream_graph/actor.rs
+++ b/src/meta/src/stream/stream_graph/actor.rs
@@ -19,8 +19,8 @@ use std::sync::Arc;
 use assert_matches::assert_matches;
 use itertools::Itertools;
 use risingwave_common::bail;
-use risingwave_common::buffer::Bitmap;
-use risingwave_common::hash::{ActorId, ActorMapping, ParallelUnitId};
+use risingwave_common::bitmap::Bitmap;
+use risingwave_common::hash::{ActorId, ActorMapping, WorkerSlotId};
 use risingwave_common::util::iter_util::ZipEqFast;
 use risingwave_pb::meta::table_fragments::Fragment;
 use risingwave_pb::plan_common::ExprContext;
@@ -330,8 +330,8 @@ impl ExternalChange {
     }
 }
 
-/// The parallel unit location of actors.
-type ActorLocations = BTreeMap<GlobalActorId, ParallelUnitId>;
+/// The worker slot location of actors.
+type ActorLocations = BTreeMap<GlobalActorId, WorkerSlotId>;
 
 /// The actual mutable state of building an actor graph.
 ///
@@ -369,7 +369,7 @@ impl ActorGraphBuildStateInner {
         &mut self,
         actor_id: GlobalActorId,
         fragment_id: GlobalFragmentId,
-        parallel_unit_id: ParallelUnitId,
+        worker_slot_id: WorkerSlotId,
         vnode_bitmap: Option<Bitmap>,
         node: Arc<StreamNode>,
     ) {
@@ -381,18 +381,14 @@ impl ActorGraphBuildStateInner {
             .unwrap();
 
         self.building_locations
-            .try_insert(actor_id, parallel_unit_id)
+            .try_insert(actor_id, worker_slot_id)
             .unwrap();
     }
 
     /// Record the location of an external actor.
-    fn record_external_location(
-        &mut self,
-        actor_id: GlobalActorId,
-        parallel_unit_id: ParallelUnitId,
-    ) {
+    fn record_external_location(&mut self, actor_id: GlobalActorId, worker_slot_id: WorkerSlotId) {
         self.external_locations
-            .try_insert(actor_id, parallel_unit_id)
+            .try_insert(actor_id, worker_slot_id)
             .unwrap();
     }
 
@@ -466,7 +462,7 @@ impl ActorGraphBuildStateInner {
 
     /// Get the location of an actor. Will look up the location map of both the actors to be built
     /// and the external actors.
-    fn get_location(&self, actor_id: GlobalActorId) -> ParallelUnitId {
+    fn get_location(&self, actor_id: GlobalActorId) -> WorkerSlotId {
         self.building_locations
             .get(&actor_id)
             .copied()
@@ -534,9 +530,9 @@ impl ActorGraphBuildStateInner {
             DispatcherType::Hash | DispatcherType::Broadcast | DispatcherType::Simple => {
                 // Add dispatchers for the upstream actors.
                 let dispatcher = if let DispatcherType::Hash = dt {
-                    // Transform the `ParallelUnitMapping` from the downstream distribution to the
+                    // Transform the `WorkerSlotMapping` from the downstream distribution to the
                     // `ActorMapping`, used for the `HashDispatcher` for the upstream actors.
-                    let downstream_locations: HashMap<ParallelUnitId, ActorId> = downstream
+                    let downstream_locations: HashMap<WorkerSlotId, ActorId> = downstream
                         .actor_ids
                         .iter()
                         .map(|&actor_id| (self.get_location(actor_id), actor_id.as_global_id()))
@@ -673,7 +669,7 @@ impl ActorGraphBuilder {
         // Schedule the distribution of all building fragments.
         let scheduler = schedule::Scheduler::new(
             streaming_job_id,
-            cluster_info.parallel_units.values().cloned(),
+            &cluster_info.worker_nodes,
             default_parallelism,
         )?;
         let distributions = scheduler.schedule(&fragment_graph)?;
@@ -699,12 +695,7 @@ impl ActorGraphBuilder {
     fn build_locations(&self, actor_locations: ActorLocations) -> Locations {
         let actor_locations = actor_locations
             .into_iter()
-            .map(|(id, p)| {
-                (
-                    id.as_global_id(),
-                    self.cluster_info.parallel_units[&p].clone(),
-                )
-            })
+            .map(|(id, worker_slot_id)| (id.as_global_id(), worker_slot_id))
             .collect();
 
         let worker_locations = self.cluster_info.worker_nodes.clone();
@@ -742,15 +733,15 @@ impl ActorGraphBuilder {
             external_locations,
         } = self.build_actor_graph(id_gen)?;
 
-        for parallel_unit_id in external_locations.values() {
-            if let Some(parallel_unit) = self
+        for worker_slot_id in external_locations.values() {
+            if self
                 .cluster_info
-                .unschedulable_parallel_units
-                .get(parallel_unit_id)
+                .unschedulable_workers
+                .contains(&worker_slot_id.worker_id())
             {
                 bail!(
-                    "The worker {} where the associated upstream is located is unscheduable",
-                    parallel_unit.worker_node_id
+                    "The worker {} where the associated upstream is located is unschedulable",
+                    worker_slot_id.worker_id(),
                 );
             }
         }
@@ -859,15 +850,15 @@ impl ActorGraphBuilder {
                 let bitmaps = distribution.as_hash().map(|m| m.to_bitmaps());
 
                 distribution
-                    .parallel_units()
-                    .map(|parallel_unit_id| {
+                    .worker_slots()
+                    .map(|worker_slot| {
                         let actor_id = state.next_actor_id();
-                        let vnode_bitmap = bitmaps.as_ref().map(|m| &m[&parallel_unit_id]).cloned();
+                        let vnode_bitmap = bitmaps.as_ref().map(|m| &m[&worker_slot]).cloned();
 
                         state.inner.add_actor(
                             actor_id,
                             fragment_id,
-                            parallel_unit_id,
+                            worker_slot,
                             vnode_bitmap,
                             node.clone(),
                         );
@@ -883,8 +874,8 @@ impl ActorGraphBuilder {
                 .iter()
                 .map(|a| {
                     let actor_id = GlobalActorId::new(a.actor_id);
-                    let parallel_unit_id = match &distribution {
-                        Distribution::Singleton(parallel_unit_id) => *parallel_unit_id,
+                    let worker_slot_id = match &distribution {
+                        Distribution::Singleton(worker_slot_id) => *worker_slot_id,
                         Distribution::Hash(mapping) => mapping
                             .get_matched(&Bitmap::from(a.get_vnode_bitmap().unwrap()))
                             .unwrap(),
@@ -892,7 +883,7 @@ impl ActorGraphBuilder {
 
                     state
                         .inner
-                        .record_external_location(actor_id, parallel_unit_id);
+                        .record_external_location(actor_id, worker_slot_id);
 
                     actor_id
                 })
diff --git a/src/meta/src/stream/stream_graph/fragment.rs b/src/meta/src/stream/stream_graph/fragment.rs
index e347dd0287f36..f6720966405e3 100644
--- a/src/meta/src/stream/stream_graph/fragment.rs
+++ b/src/meta/src/stream/stream_graph/fragment.rs
@@ -38,8 +38,8 @@ use risingwave_pb::stream_plan::{
     StreamFragmentGraph as StreamFragmentGraphProto, StreamNode, StreamScanType,
 };
 
-use crate::manager::{DdlType, IdGenManagerImpl, MetaSrvEnv, StreamingJob};
-use crate::model::FragmentId;
+use crate::manager::{DdlType, IdGenManagerImpl, MetaSrvEnv, StreamingJob, WorkerId};
+use crate::model::{ActorId, FragmentId};
 use crate::stream::stream_graph::id::{GlobalFragmentId, GlobalFragmentIdGen, GlobalTableIdGen};
 use crate::stream::stream_graph::schedule::Distribution;
 use crate::MetaResult;
@@ -311,7 +311,7 @@ pub struct StreamFragmentGraph {
     dependent_table_ids: HashSet<TableId>,
 
     /// The default parallelism of the job, specified by the `STREAMING_PARALLELISM` session
-    /// variable. If not specified, all active parallel units will be used.
+    /// variable. If not specified, all active worker slots will be used.
     specified_parallelism: Option<NonZeroUsize>,
 }
 
@@ -539,6 +539,9 @@ pub struct CompleteStreamFragmentGraph {
     /// The required information of existing fragments.
     existing_fragments: HashMap<GlobalFragmentId, Fragment>,
 
+    /// The location of the actors in the existing fragments.
+    existing_actor_location: HashMap<ActorId, WorkerId>,
+
     /// Extra edges between existing fragments and the building fragments.
     extra_downstreams: HashMap<GlobalFragmentId, HashMap<GlobalFragmentId, StreamFragmentEdge>>,
 
@@ -550,11 +553,13 @@ pub struct FragmentGraphUpstreamContext {
     /// Root fragment is the root of upstream stream graph, which can be a
     /// mview fragment or source fragment for cdc source job
     upstream_root_fragments: HashMap<TableId, Fragment>,
+    upstream_actor_location: HashMap<ActorId, WorkerId>,
 }
 
 pub struct FragmentGraphDownstreamContext {
     original_table_fragment_id: FragmentId,
     downstream_fragments: Vec<(DispatchStrategy, Fragment)>,
+    downstream_actor_location: HashMap<ActorId, WorkerId>,
 }
 
 impl CompleteStreamFragmentGraph {
@@ -565,6 +570,7 @@ impl CompleteStreamFragmentGraph {
         Self {
             building_graph: graph,
             existing_fragments: Default::default(),
+            existing_actor_location: Default::default(),
             extra_downstreams: Default::default(),
             extra_upstreams: Default::default(),
         }
@@ -575,12 +581,14 @@ impl CompleteStreamFragmentGraph {
     pub fn with_upstreams(
         graph: StreamFragmentGraph,
         upstream_root_fragments: HashMap<TableId, Fragment>,
+        existing_actor_location: HashMap<ActorId, u32>,
         ddl_type: DdlType,
     ) -> MetaResult<Self> {
         Self::build_helper(
             graph,
             Some(FragmentGraphUpstreamContext {
                 upstream_root_fragments,
+                upstream_actor_location: existing_actor_location,
             }),
             None,
             ddl_type,
@@ -593,6 +601,7 @@ impl CompleteStreamFragmentGraph {
         graph: StreamFragmentGraph,
         original_table_fragment_id: FragmentId,
         downstream_fragments: Vec<(DispatchStrategy, Fragment)>,
+        existing_actor_location: HashMap<ActorId, u32>,
         ddl_type: DdlType,
     ) -> MetaResult<Self> {
         Self::build_helper(
@@ -601,6 +610,32 @@ impl CompleteStreamFragmentGraph {
             Some(FragmentGraphDownstreamContext {
                 original_table_fragment_id,
                 downstream_fragments,
+                downstream_actor_location: existing_actor_location,
+            }),
+            ddl_type,
+        )
+    }
+
+    /// For replacing an existing table based on shared cdc source
+    pub fn with_upstreams_and_downstreams(
+        graph: StreamFragmentGraph,
+        upstream_root_fragments: HashMap<TableId, Fragment>,
+        upstream_actor_location: HashMap<ActorId, u32>,
+        original_table_fragment_id: FragmentId,
+        downstream_fragments: Vec<(DispatchStrategy, Fragment)>,
+        downstream_actor_location: HashMap<ActorId, u32>,
+        ddl_type: DdlType,
+    ) -> MetaResult<Self> {
+        Self::build_helper(
+            graph,
+            Some(FragmentGraphUpstreamContext {
+                upstream_root_fragments,
+                upstream_actor_location,
+            }),
+            Some(FragmentGraphDownstreamContext {
+                original_table_fragment_id,
+                downstream_fragments,
+                downstream_actor_location,
             }),
             ddl_type,
         )
@@ -617,8 +652,11 @@ impl CompleteStreamFragmentGraph {
         let mut extra_upstreams = HashMap::new();
         let mut existing_fragments = HashMap::new();
 
+        let mut existing_actor_location = HashMap::new();
+
         if let Some(FragmentGraphUpstreamContext {
             upstream_root_fragments,
+            upstream_actor_location,
         }) = upstream_ctx
         {
             for (&id, fragment) in &mut graph.fragments {
@@ -781,11 +819,14 @@ impl CompleteStreamFragmentGraph {
                     .into_values()
                     .map(|f| (GlobalFragmentId::new(f.fragment_id), f)),
             );
+
+            existing_actor_location.extend(upstream_actor_location);
         }
 
         if let Some(FragmentGraphDownstreamContext {
             original_table_fragment_id,
             downstream_fragments,
+            downstream_actor_location,
         }) = downstream_ctx
         {
             let original_table_fragment_id = GlobalFragmentId::new(original_table_fragment_id);
@@ -821,11 +862,14 @@ impl CompleteStreamFragmentGraph {
                     .into_iter()
                     .map(|(_, f)| (GlobalFragmentId::new(f.fragment_id), f)),
             );
+
+            existing_actor_location.extend(downstream_actor_location);
         }
 
         Ok(Self {
             building_graph: graph,
             existing_fragments,
+            existing_actor_location,
             extra_downstreams,
             extra_upstreams,
         })
@@ -886,7 +930,12 @@ impl CompleteStreamFragmentGraph {
     pub(super) fn existing_distribution(&self) -> HashMap<GlobalFragmentId, Distribution> {
         self.existing_fragments
             .iter()
-            .map(|(&id, f)| (id, Distribution::from_fragment(f)))
+            .map(|(&id, f)| {
+                (
+                    id,
+                    Distribution::from_fragment(f, &self.existing_actor_location),
+                )
+            })
             .collect()
     }
 
@@ -973,7 +1022,6 @@ impl CompleteStreamFragmentGraph {
             fragment_type_mask: inner.fragment_type_mask,
             distribution_type,
             actors,
-            vnode_mapping: Some(distribution.into_mapping().to_protobuf()),
             state_table_ids,
             upstream_fragment_ids,
         }
diff --git a/src/meta/src/stream/stream_graph/schedule.rs b/src/meta/src/stream/stream_graph/schedule.rs
index ed2dac5be0e06..0f9e473c26486 100644
--- a/src/meta/src/stream/stream_graph/schedule.rs
+++ b/src/meta/src/stream/stream_graph/schedule.rs
@@ -18,15 +18,16 @@
     reason = "generated by crepe"
 )]
 
-use std::collections::{BTreeMap, BTreeSet, HashMap};
+use std::collections::{BTreeMap, HashMap};
 use std::num::NonZeroUsize;
 
 use either::Either;
 use enum_as_inner::EnumAsInner;
 use itertools::Itertools;
-use risingwave_common::bail;
-use risingwave_common::hash::{ParallelUnitId, ParallelUnitMapping};
-use risingwave_pb::common::{ActorInfo, ParallelUnit};
+use risingwave_common::bitmap::Bitmap;
+use risingwave_common::hash::{ActorMapping, WorkerSlotId, WorkerSlotMapping};
+use risingwave_common::{bail, hash};
+use risingwave_pb::common::{ActorInfo, WorkerNode};
 use risingwave_pb::meta::table_fragments::fragment::{
     FragmentDistributionType, PbFragmentDistributionType,
 };
@@ -46,7 +47,7 @@ type HashMappingId = usize;
 /// See [`Distribution`] for the public interface.
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
 enum DistId {
-    Singleton(ParallelUnitId),
+    Singleton(WorkerSlotId),
     Hash(HashMappingId),
 }
 
@@ -64,7 +65,7 @@ enum Fact {
     /// A distribution requirement for an external(existing) fragment.
     ExternalReq { id: Id, dist: DistId },
     /// A singleton requirement for a building fragment.
-    /// Note that the physical parallel unit is not determined yet.
+    /// Note that the physical worker slot is not determined yet.
     SingletonReq(Id),
 }
 
@@ -73,7 +74,7 @@ enum Fact {
 enum Result {
     /// This fragment is required to be distributed by the given [`DistId`].
     Required(DistId),
-    /// This fragment is singleton, and should be scheduled to the default parallel unit.
+    /// This fragment is singleton, and should be scheduled to the default worker slot.
     DefaultSingleton,
     /// This fragment is hash-distributed, and should be scheduled by the default hash mapping.
     DefaultHash,
@@ -129,49 +130,57 @@ crepe::crepe! {
 /// The distribution of a fragment.
 #[derive(Debug, Clone, EnumAsInner)]
 pub(super) enum Distribution {
-    /// The fragment is singleton and is scheduled to the given parallel unit.
-    Singleton(ParallelUnitId),
+    /// The fragment is singleton and is scheduled to the given worker slot.
+    Singleton(WorkerSlotId),
 
     /// The fragment is hash-distributed and is scheduled by the given hash mapping.
-    Hash(ParallelUnitMapping),
+    Hash(WorkerSlotMapping),
 }
 
 impl Distribution {
     /// The parallelism required by the distribution.
     pub fn parallelism(&self) -> usize {
-        self.parallel_units().count()
+        self.worker_slots().count()
     }
 
-    /// All parallel units required by the distribution.
-    pub fn parallel_units(&self) -> impl Iterator<Item = ParallelUnitId> + '_ {
+    /// All worker slots required by the distribution.
+    pub fn worker_slots(&self) -> impl Iterator<Item = WorkerSlotId> + '_ {
         match self {
             Distribution::Singleton(p) => Either::Left(std::iter::once(*p)),
             Distribution::Hash(mapping) => Either::Right(mapping.iter_unique()),
         }
     }
 
-    /// Convert the distribution to a [`ParallelUnitMapping`].
-    ///
-    /// - For singleton distribution, all of the virtual nodes are mapped to the same parallel unit.
-    /// - For hash distribution, the mapping is returned as is.
-    pub fn into_mapping(self) -> ParallelUnitMapping {
-        match self {
-            Distribution::Singleton(p) => ParallelUnitMapping::new_single(p),
-            Distribution::Hash(mapping) => mapping,
-        }
-    }
-
     /// Create a distribution from a persisted protobuf `Fragment`.
-    pub fn from_fragment(fragment: &risingwave_pb::meta::table_fragments::Fragment) -> Self {
-        let mapping = ParallelUnitMapping::from_protobuf(fragment.get_vnode_mapping().unwrap());
-
+    pub fn from_fragment(
+        fragment: &risingwave_pb::meta::table_fragments::Fragment,
+        actor_location: &HashMap<ActorId, WorkerId>,
+    ) -> Self {
         match fragment.get_distribution_type().unwrap() {
             FragmentDistributionType::Unspecified => unreachable!(),
             FragmentDistributionType::Single => {
-                let parallel_unit = mapping.to_single().unwrap();
-                Distribution::Singleton(parallel_unit)
+                let actor_id = fragment.actors.iter().exactly_one().unwrap().actor_id;
+                let location = actor_location.get(&actor_id).unwrap();
+                let worker_slot_id = WorkerSlotId::new(*location, 0);
+                Distribution::Singleton(worker_slot_id)
+            }
+            FragmentDistributionType::Hash => {
+                let actor_bitmaps: HashMap<_, _> = fragment
+                    .actors
+                    .iter()
+                    .map(|actor| {
+                        (
+                            actor.actor_id as hash::ActorId,
+                            Bitmap::from(actor.vnode_bitmap.as_ref().unwrap()),
+                        )
+                    })
+                    .collect();
+
+                let actor_mapping = ActorMapping::from_bitmaps(&actor_bitmaps);
+                let mapping = actor_mapping.to_worker_slot(actor_location);
+
+                Distribution::Hash(mapping)
             }
-            FragmentDistributionType::Hash => Distribution::Hash(mapping),
         }
     }
 
@@ -187,54 +196,55 @@ impl Distribution {
 /// [`Scheduler`] schedules the distribution of fragments in a stream graph.
 pub(super) struct Scheduler {
     /// The default hash mapping for hash-distributed fragments, if there's no requirement derived.
-    default_hash_mapping: ParallelUnitMapping,
+    default_hash_mapping: WorkerSlotMapping,
 
-    /// The default parallel unit for singleton fragments, if there's no requirement derived.
-    default_singleton_parallel_unit: ParallelUnitId,
+    /// The default worker slot for singleton fragments, if there's no requirement derived.
+    default_singleton_worker_slot: WorkerSlotId,
 }
 
 impl Scheduler {
-    /// Create a new [`Scheduler`] with the given parallel units and the default parallelism.
+    /// Create a new [`Scheduler`] with the given worker slots and the default parallelism.
     ///
     /// Each hash-distributed fragment will be scheduled to at most `default_parallelism` parallel
     /// units, in a round-robin fashion on all compute nodes. If the `default_parallelism` is
-    /// `None`, all parallel units will be used.
+    /// `None`, all worker slots will be used.
     ///
     /// For different streaming jobs, we even out possible scheduling skew by using the streaming job id as the salt for the scheduling algorithm.
     pub fn new(
         streaming_job_id: u32,
-        parallel_units: impl IntoIterator<Item = ParallelUnit>,
+        workers: &HashMap<u32, WorkerNode>,
         default_parallelism: NonZeroUsize,
     ) -> MetaResult<Self> {
-        // Group parallel units with worker node.
-        let mut slots = BTreeMap::new();
-        for parallel_unit in parallel_units {
-            slots
-                .entry(parallel_unit.worker_node_id as WorkerId)
-                .or_insert_with(BTreeSet::new)
-                .insert(parallel_unit.id as ParallelUnitId);
-        }
+        // Group worker slots with worker node.
+
+        let slots = workers
+            .iter()
+            .map(|(worker_id, worker)| (*worker_id, worker.parallelism as usize))
+            .collect();
 
         let parallelism = default_parallelism.get();
         let scheduled = schedule_units_for_slots(&slots, parallelism, streaming_job_id)?;
 
-        let scheduled_parallel_units = scheduled.values().flatten().cloned().sorted().collect_vec();
-        assert_eq!(scheduled_parallel_units.len(), parallelism);
+        let scheduled_worker_slots = scheduled
+            .into_iter()
+            .flat_map(|(worker_id, size)| {
+                (0..size).map(move |slot| WorkerSlotId::new(worker_id, slot))
+            })
+            .collect_vec();
+
+        assert_eq!(scheduled_worker_slots.len(), parallelism);
 
         // Build the default hash mapping uniformly.
-        let default_hash_mapping = ParallelUnitMapping::build_from_ids(&scheduled_parallel_units);
+        let default_hash_mapping = WorkerSlotMapping::build_from_ids(&scheduled_worker_slots);
 
         let single_scheduled = schedule_units_for_slots(&slots, 1, streaming_job_id)?;
-        let default_singleton_parallel_unit = single_scheduled
-            .values()
-            .flatten()
-            .exactly_one()
-            .cloned()
-            .unwrap();
+        let default_single_worker_id = single_scheduled.keys().exactly_one().cloned().unwrap();
+
+        let default_singleton_worker_slot = WorkerSlotId::new(default_single_worker_id, 0);
 
         Ok(Self {
             default_hash_mapping,
-            default_singleton_parallel_unit,
+            default_singleton_worker_slot,
         })
     }
 
@@ -271,7 +281,7 @@ impl Scheduler {
         // External
         for (id, req) in existing_distribution {
             let dist = match req {
-                Distribution::Singleton(parallel_unit) => DistId::Singleton(parallel_unit),
+                Distribution::Singleton(worker_slot_id) => DistId::Singleton(worker_slot_id),
                 Distribution::Hash(mapping) => DistId::Hash(hash_mapping_id[&mapping]),
             };
             facts.push(Fact::ExternalReq { id, dist });
@@ -301,8 +311,8 @@ impl Scheduler {
             .map(|Success(id, result)| {
                 let distribution = match result {
                     // Required
-                    Result::Required(DistId::Singleton(parallel_unit)) => {
-                        Distribution::Singleton(parallel_unit)
+                    Result::Required(DistId::Singleton(worker_slot)) => {
+                        Distribution::Singleton(worker_slot)
                     }
                     Result::Required(DistId::Hash(mapping)) => {
                         Distribution::Hash(all_hash_mappings[mapping].clone())
@@ -310,7 +320,7 @@ impl Scheduler {
 
                     // Default
                     Result::DefaultSingleton => {
-                        Distribution::Singleton(self.default_singleton_parallel_unit)
+                        Distribution::Singleton(self.default_singleton_worker_slot)
                     }
                     Result::DefaultHash => Distribution::Hash(self.default_hash_mapping.clone()),
                 };
@@ -324,11 +334,11 @@ impl Scheduler {
     }
 }
 
-/// [`Locations`] represents the parallel unit and worker locations of the actors.
+/// [`Locations`] represents the worker slot and worker locations of the actors.
 #[cfg_attr(test, derive(Default))]
 pub struct Locations {
     /// actor location map.
-    pub actor_locations: BTreeMap<ActorId, ParallelUnit>,
+    pub actor_locations: BTreeMap<ActorId, WorkerSlotId>,
     /// worker location map.
     pub worker_locations: WorkerLocations,
 }
@@ -338,7 +348,7 @@ impl Locations {
     pub fn worker_actors(&self) -> HashMap<WorkerId, Vec<ActorId>> {
         self.actor_locations
             .iter()
-            .map(|(actor_id, parallel_unit)| (parallel_unit.worker_node_id, *actor_id))
+            .map(|(actor_id, worker_slot_id)| (worker_slot_id.worker_id(), *actor_id))
             .into_group_map()
     }
 
@@ -346,9 +356,9 @@ impl Locations {
     pub fn actor_infos(&self) -> impl Iterator<Item = ActorInfo> + '_ {
         self.actor_locations
             .iter()
-            .map(|(actor_id, parallel_unit)| ActorInfo {
+            .map(|(actor_id, worker_slot_id)| ActorInfo {
                 actor_id: *actor_id,
-                host: self.worker_locations[&parallel_unit.worker_node_id]
+                host: self.worker_locations[&worker_slot_id.worker_id()]
                     .host
                     .clone(),
             })
@@ -425,7 +435,7 @@ mod tests {
             Fact::Fragment(103.into()),
             Fact::Fragment(104.into()),
             Fact::ExternalReq { id: 1.into(), dist: DistId::Hash(1) },
-            Fact::ExternalReq { id: 2.into(), dist: DistId::Singleton(2) },
+            Fact::ExternalReq { id: 2.into(), dist: DistId::Singleton(WorkerSlotId::new(0, 2)) },
             Fact::Edge { from: 1.into(), to: 101.into(), dt: NoShuffle },
             Fact::Edge { from: 2.into(), to: 102.into(), dt: NoShuffle },
             Fact::Edge { from: 101.into(), to: 103.into(), dt: Hash },
@@ -435,7 +445,7 @@ mod tests {
 
         let expected = maplit::hashmap! {
             101.into() => Result::Required(DistId::Hash(1)),
-            102.into() => Result::Required(DistId::Singleton(2)),
+            102.into() => Result::Required(DistId::Singleton(WorkerSlotId::new(0, 2))),
             103.into() => Result::DefaultHash,
             104.into() => Result::DefaultSingleton,
         };
diff --git a/src/meta/src/stream/stream_manager.rs b/src/meta/src/stream/stream_manager.rs
index a7ba22b914661..050238e2dbc6d 100644
--- a/src/meta/src/stream/stream_manager.rs
+++ b/src/meta/src/stream/stream_manager.rs
@@ -17,6 +17,7 @@ use std::sync::Arc;
 
 use futures::future::join_all;
 use itertools::Itertools;
+use risingwave_common::bail;
 use risingwave_common::catalog::TableId;
 use risingwave_meta_model_v2::ObjectId;
 use risingwave_pb::catalog::{CreateType, Subscription, Table};
@@ -28,8 +29,10 @@ use tokio::sync::{oneshot, Mutex};
 use tracing::Instrument;
 
 use super::{Locations, RescheduleOptions, ScaleControllerRef, TableResizePolicy};
-use crate::barrier::{BarrierScheduler, Command, ReplaceTablePlan, StreamRpcManager};
-use crate::manager::{DdlType, MetaSrvEnv, MetadataManager, StreamingJob};
+use crate::barrier::{
+    BarrierScheduler, Command, CreateStreamingJobCommandInfo, ReplaceTablePlan, StreamRpcManager,
+};
+use crate::manager::{DdlType, MetaSrvEnv, MetadataManager, NotificationVersion, StreamingJob};
 use crate::model::{ActorId, FragmentId, MetadataModel, TableFragments, TableParallelism};
 use crate::stream::{to_build_actor_info, SourceManagerRef};
 use crate::{MetaError, MetaResult};
@@ -44,7 +47,6 @@ pub struct CreateStreamingJobOption {
 /// [`CreateStreamingJobContext`] carries one-time infos for creating a streaming job.
 ///
 /// Note: for better readability, keep this struct complete and immutable once created.
-#[cfg_attr(test, derive(Default))]
 pub struct CreateStreamingJobContext {
     /// New dispatchers to add from upstream actors to downstream actors.
     pub dispatchers: HashMap<ActorId, Vec<Dispatcher>>,
@@ -76,6 +78,8 @@ pub struct CreateStreamingJobContext {
     pub replace_table_job_info: Option<(StreamingJob, ReplaceTableContext, TableFragments)>,
 
     pub option: CreateStreamingJobOption,
+
+    pub streaming_job: StreamingJob,
 }
 
 impl CreateStreamingJobContext {
@@ -88,7 +92,7 @@ pub enum CreatingState {
     Failed { reason: MetaError },
     // sender is used to notify the canceling result.
     Canceling { finish_tx: oneshot::Sender<()> },
-    Created,
+    Created { version: NotificationVersion },
 }
 
 struct StreamingJobExecution {
@@ -174,6 +178,10 @@ pub struct ReplaceTableContext {
 
     /// The locations of the existing actors, essentially the downstream chain actors to update.
     pub existing_locations: Locations,
+
+    pub streaming_job: StreamingJob,
+
+    pub dummy_id: u32,
 }
 
 /// `GlobalStreamManager` manages all the streams in the system.
@@ -228,7 +236,7 @@ impl GlobalStreamManager {
         self: &Arc<Self>,
         table_fragments: TableFragments,
         ctx: CreateStreamingJobContext,
-    ) -> MetaResult<()> {
+    ) -> MetaResult<NotificationVersion> {
         let table_id = table_fragments.table_id();
         let (sender, mut receiver) = tokio::sync::mpsc::channel(10);
         let execution = StreamingJobExecution::new(table_id, sender.clone());
@@ -237,12 +245,12 @@ impl GlobalStreamManager {
         let stream_manager = self.clone();
         let fut = async move {
             let res = stream_manager
-                .create_streaming_job_impl( table_fragments, ctx)
+                .create_streaming_job_impl(table_fragments, ctx)
                 .await;
             match res {
-                Ok(_) => {
+                Ok(version) => {
                     let _ = sender
-                        .send(CreatingState::Created)
+                        .send(CreatingState::Created { version })
                         .await
                         .inspect_err(|_| tracing::warn!("failed to notify created: {table_id}"));
                 }
@@ -261,72 +269,66 @@ impl GlobalStreamManager {
         .in_current_span();
         tokio::spawn(fut);
 
-        let res = try {
-            while let Some(state) = receiver.recv().await {
-                match state {
-                    CreatingState::Failed { reason } => {
-                        tracing::debug!(id=?table_id, "stream job failed");
-                        self.creating_job_info.delete_job(table_id).await;
-                        return Err(reason);
-                    }
-                    CreatingState::Canceling { finish_tx } => {
-                        tracing::debug!(id=?table_id, "cancelling streaming job");
-                        if let Ok(table_fragments) = self
-                            .metadata_manager
-                            .get_job_fragments_by_id(&table_id)
-                            .await
-                        {
-                            // try to cancel buffered creating command.
-                            if self.barrier_scheduler.try_cancel_scheduled_create(table_id) {
-                                tracing::debug!(
-                                    "cancelling streaming job {table_id} in buffer queue."
-                                );
-                                let node_actors = table_fragments.worker_actor_ids();
-                                let cluster_info =
-                                    self.metadata_manager.get_streaming_cluster_info().await?;
-                                self.stream_rpc_manager
-                                    .drop_actors(
-                                        &cluster_info.worker_nodes,
-                                        node_actors.into_iter(),
-                                    )
-                                    .await?;
-
-                                if let MetadataManager::V1(mgr) = &self.metadata_manager {
-                                    mgr.fragment_manager
-                                        .drop_table_fragments_vec(&HashSet::from_iter(
-                                            std::iter::once(table_id),
-                                        ))
-                                        .await?;
-                                }
-                            } else if !table_fragments.is_created() {
-                                tracing::debug!(
-                                    "cancelling streaming job {table_id} by issue cancel command."
-                                );
-
-                                self.barrier_scheduler
-                                    .run_command(Command::CancelStreamingJob(table_fragments))
+        while let Some(state) = receiver.recv().await {
+            match state {
+                CreatingState::Failed { reason } => {
+                    tracing::debug!(id=?table_id, "stream job failed");
+                    // FIXME(kwannoel): For creating stream jobs
+                    // we need to clean up the resources in the stream manager.
+                    self.creating_job_info.delete_job(table_id).await;
+                    return Err(reason);
+                }
+                CreatingState::Canceling { finish_tx } => {
+                    tracing::debug!(id=?table_id, "cancelling streaming job");
+                    if let Ok(table_fragments) = self
+                        .metadata_manager
+                        .get_job_fragments_by_id(&table_id)
+                        .await
+                    {
+                        // try to cancel buffered creating command.
+                        if self.barrier_scheduler.try_cancel_scheduled_create(table_id) {
+                            tracing::debug!("cancelling streaming job {table_id} in buffer queue.");
+                            let node_actors = table_fragments.worker_actor_ids();
+                            let cluster_info =
+                                self.metadata_manager.get_streaming_cluster_info().await?;
+                            self.stream_rpc_manager
+                                .drop_actors(&cluster_info.worker_nodes, node_actors.into_iter())
+                                .await?;
+
+                            if let MetadataManager::V1(mgr) = &self.metadata_manager {
+                                mgr.fragment_manager
+                                    .drop_table_fragments_vec(&HashSet::from_iter(std::iter::once(
+                                        table_id,
+                                    )))
                                     .await?;
-                            } else {
-                                // streaming job is already completed.
-                                continue;
                             }
-                            let _ = finish_tx.send(()).inspect_err(|_| {
-                                tracing::warn!("failed to notify cancelled: {table_id}")
-                            });
-                            self.creating_job_info.delete_job(table_id).await;
-                            return Err(MetaError::cancelled("create"));
+                        } else if !table_fragments.is_created() {
+                            tracing::debug!(
+                                "cancelling streaming job {table_id} by issue cancel command."
+                            );
+
+                            self.barrier_scheduler
+                                .run_command(Command::CancelStreamingJob(table_fragments))
+                                .await?;
+                        } else {
+                            // streaming job is already completed.
+                            continue;
                         }
-                    }
-                    CreatingState::Created => {
+                        let _ = finish_tx.send(()).inspect_err(|_| {
+                            tracing::warn!("failed to notify cancelled: {table_id}")
+                        });
                         self.creating_job_info.delete_job(table_id).await;
-                        return Ok(());
+                        return Err(MetaError::cancelled("create"));
                     }
                 }
+                CreatingState::Created { version } => {
+                    self.creating_job_info.delete_job(table_id).await;
+                    return Ok(version);
+                }
             }
-        };
-
+        }
         self.creating_job_info.delete_job(table_id).await;
-        res
+        bail!("receiver failed to get notification version for finished stream job")
     }
 
     async fn build_actors(
@@ -334,6 +336,7 @@ impl GlobalStreamManager {
         table_fragments: &TableFragments,
         building_locations: &Locations,
         existing_locations: &Locations,
+        subscription_depend_table_id: TableId,
     ) -> MetaResult<()> {
         let actor_map = table_fragments.actor_map();
 
@@ -367,7 +370,7 @@ impl GlobalStreamManager {
                             to_build_actor_info(
                                 actor_map[actor_id].clone(),
                                 &subscriptions,
-                                table_fragments.table_id(),
+                                subscription_depend_table_id,
                             )
                         })
                         .collect::<Vec<_>>();
@@ -392,6 +395,7 @@ impl GlobalStreamManager {
         &self,
         table_fragments: TableFragments,
         CreateStreamingJobContext {
+            streaming_job,
             dispatchers,
             upstream_root_actors,
             building_locations,
@@ -400,14 +404,20 @@ impl GlobalStreamManager {
             create_type,
             ddl_type,
             replace_table_job_info,
+            internal_tables,
             ..
         }: CreateStreamingJobContext,
-    ) -> MetaResult<()> {
+    ) -> MetaResult<NotificationVersion> {
         let mut replace_table_command = None;
         let mut replace_table_id = None;
 
-        self.build_actors(&table_fragments, &building_locations, &existing_locations)
-            .await?;
+        self.build_actors(
+            &table_fragments,
+            &building_locations,
+            &existing_locations,
+            table_fragments.table_id(),
+        )
+        .await?;
         tracing::debug!(
             table_id = %table_fragments.table_id(),
             "built actors finished"
@@ -418,6 +428,7 @@ impl GlobalStreamManager {
                 &table_fragments,
                 &context.building_locations,
                 &context.existing_locations,
+                context.old_table_fragments.table_id(),
             )
             .await?;
 
@@ -439,15 +450,17 @@ impl GlobalStreamManager {
             let init_split_assignment =
                 self.source_manager.allocate_splits(&dummy_table_id).await?;
 
+            replace_table_id = Some(dummy_table_id);
+
             replace_table_command = Some(ReplaceTablePlan {
                 old_table_fragments: context.old_table_fragments,
                 new_table_fragments: table_fragments,
                 merge_updates: context.merge_updates,
                 dispatchers: context.dispatchers,
                 init_split_assignment,
+                streaming_job,
+                dummy_id: dummy_table_id.table_id,
             });
-
-            replace_table_id = Some(dummy_table_id);
         }
 
         let table_id = table_fragments.table_id();
@@ -463,34 +476,48 @@ impl GlobalStreamManager {
                 .await?,
         );
 
-        let command = Command::CreateStreamingJob {
+        let info = CreateStreamingJobCommandInfo {
             table_fragments,
             upstream_root_actors,
             dispatchers,
             init_split_assignment,
             definition: definition.to_string(),
+            streaming_job: streaming_job.clone(),
+            internal_tables: internal_tables.into_values().collect_vec(),
             ddl_type,
-            replace_table: replace_table_command,
             create_type,
         };
+
+        let command = Command::CreateStreamingJob {
+            info,
+            replace_table: replace_table_command,
+        };
         tracing::debug!("sending Command::CreateStreamingJob");
-        if let Err(err) = self.barrier_scheduler.run_command(command).await {
-            if create_type == CreateType::Foreground || err.is_cancelled() {
-                let mut table_ids = HashSet::from_iter(std::iter::once(table_id));
-                if let Some(dummy_table_id) = replace_table_id {
-                    table_ids.insert(dummy_table_id);
-                }
-                if let MetadataManager::V1(mgr) = &self.metadata_manager {
-                    mgr.fragment_manager
-                        .drop_table_fragments_vec(&table_ids)
-                        .await?;
+        let result: MetaResult<NotificationVersion> = try {
+            self.barrier_scheduler.run_command(command).await?;
+            tracing::debug!("first barrier collected for stream job");
+            self.metadata_manager
+                .wait_streaming_job_finished(&streaming_job)
+                .await?
+        };
+        match result {
+            Err(err) => {
+                if create_type == CreateType::Foreground || err.is_cancelled() {
+                    let mut table_ids = HashSet::from_iter(std::iter::once(table_id));
+                    if let Some(dummy_table_id) = replace_table_id {
+                        table_ids.insert(dummy_table_id);
+                    }
+                    if let MetadataManager::V1(mgr) = &self.metadata_manager {
+                        mgr.fragment_manager
+                            .drop_table_fragments_vec(&table_ids)
+                            .await?;
+                    }
                 }
-            }
 
-            return Err(err);
+                Err(err)
+            }
+            Ok(version) => Ok(version),
         }
-
-        Ok(())
     }
 
     pub async fn replace_table(
@@ -502,10 +529,17 @@ impl GlobalStreamManager {
             dispatchers,
             building_locations,
             existing_locations,
+            dummy_id,
+            streaming_job,
         }: ReplaceTableContext,
     ) -> MetaResult<()> {
-        self.build_actors(&table_fragments, &building_locations, &existing_locations)
-            .await?;
+        self.build_actors(
+            &table_fragments,
+            &building_locations,
+            &existing_locations,
+            old_table_fragments.table_id(),
+        )
+        .await?;
 
         let dummy_table_id = table_fragments.table_id();
         let init_split_assignment = self.source_manager.allocate_splits(&dummy_table_id).await?;
@@ -518,6 +552,8 @@ impl GlobalStreamManager {
                 merge_updates,
                 dispatchers,
                 init_split_assignment,
+                dummy_id,
+                streaming_job,
             }))
             .await
             && let MetadataManager::V1(mgr) = &self.metadata_manager
@@ -785,7 +821,8 @@ mod tests {
     use std::time::Duration;
 
     use futures::{Stream, TryStreamExt};
-    use risingwave_common::hash::ParallelUnitMapping;
+    use risingwave_common::hash;
+    use risingwave_common::hash::{ActorMapping, WorkerSlotId};
     use risingwave_common::system_param::reader::SystemParamsRead;
     use risingwave_pb::common::{HostAddress, WorkerType};
     use risingwave_pb::meta::add_worker_node_request::Property;
@@ -804,7 +841,6 @@ mod tests {
     #[cfg(feature = "failpoints")]
     use tokio::sync::Notify;
     use tokio::task::JoinHandle;
-    use tokio::time::sleep;
     use tokio_stream::wrappers::UnboundedReceiverStream;
     use tonic::{Request, Response, Status, Streaming};
 
@@ -814,7 +850,7 @@ mod tests {
     use crate::manager::sink_coordination::SinkCoordinatorManager;
     use crate::manager::{
         CatalogManager, CatalogManagerRef, ClusterManager, FragmentManager, FragmentManagerRef,
-        RelationIdEnum, StreamingClusterInfo,
+        RelationIdEnum, StreamingClusterInfo, WorkerId,
     };
     use crate::rpc::ddl_controller::DropMode;
     use crate::rpc::metrics::MetaMetrics;
@@ -828,6 +864,7 @@ mod tests {
     }
 
     struct FakeStreamService {
+        worker_id: WorkerId,
         inner: Arc<FakeFragmentState>,
     }
 
@@ -895,6 +932,7 @@ mod tests {
             let (tx, rx) = unbounded_channel();
             let mut request_stream = request.into_inner();
             let inner = self.inner.clone();
+            let worker_id = self.worker_id;
             let _join_handle = spawn(async move {
                 while let Ok(Some(request)) = request_stream.try_next().await {
                     match request.request.unwrap() {
@@ -908,15 +946,21 @@ mod tests {
                                 )),
                             }));
                         }
-                        streaming_control_stream_request::Request::InjectBarrier(_) => {
+                        streaming_control_stream_request::Request::InjectBarrier(req) => {
                             let _ = tx.send(Ok(StreamingControlStreamResponse {
                                 response: Some(
                                     streaming_control_stream_response::Response::CompleteBarrier(
-                                        BarrierCompleteResponse::default(),
+                                        BarrierCompleteResponse {
+                                            epoch: req.barrier.unwrap().epoch.unwrap().prev,
+                                            worker_id,
+                                            partial_graph_id: req.partial_graph_id,
+                                            ..BarrierCompleteResponse::default()
+                                        },
                                     ),
                                 ),
                             }));
                         }
+                        streaming_control_stream_request::Request::RemovePartialGraph(..) => {}
                     }
                 }
             });
@@ -948,21 +992,7 @@ mod tests {
                 actor_infos: Mutex::new(HashMap::new()),
             });
 
-            let fake_service = FakeStreamService {
-                inner: state.clone(),
-            };
-
             let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel::<()>();
-            let stream_srv = StreamServiceServer::new(fake_service);
-            let join_handle = tokio::spawn(async move {
-                tonic::transport::Server::builder()
-                    .add_service(stream_srv)
-                    .serve_with_shutdown(addr, async move { shutdown_rx.await.unwrap() })
-                    .await
-                    .unwrap();
-            });
-
-            sleep(Duration::from_secs(1)).await;
 
             let env = MetaSrvEnv::for_test_opts(MetaOpts::test(enable_recovery)).await;
             let system_params = env.system_params_reader().await;
@@ -974,7 +1004,7 @@ mod tests {
                 port: port as i32,
             };
             let fake_parallelism = 4;
-            cluster_manager
+            let worker_node = cluster_manager
                 .add_worker_node(
                     WorkerType::ComputeNode,
                     host.clone(),
@@ -989,6 +1019,19 @@ mod tests {
                 .await?;
             cluster_manager.activate_worker_node(host).await?;
 
+            let fake_service = FakeStreamService {
+                worker_id: worker_node.id,
+                inner: state.clone(),
+            };
+            let stream_srv = StreamServiceServer::new(fake_service);
+            let join_handle = tokio::spawn(async move {
+                tonic::transport::Server::builder()
+                    .add_service(stream_srv)
+                    .serve_with_shutdown(addr, async move { shutdown_rx.await.unwrap() })
+                    .await
+                    .unwrap();
+            });
+
             let catalog_manager = Arc::new(CatalogManager::new(env.clone()).await?);
             let fragment_manager = Arc::new(FragmentManager::new(env.clone()).await?);
 
@@ -1047,7 +1090,8 @@ mod tests {
                 meta_metrics.clone(),
                 stream_rpc_manager.clone(),
                 scale_controller.clone(),
-            );
+            )
+            .await;
 
             let stream_manager = GlobalStreamManager::new(
                 env.clone(),
@@ -1085,24 +1129,22 @@ mod tests {
             table_id: TableId,
             fragments: BTreeMap<FragmentId, Fragment>,
         ) -> MetaResult<()> {
-            // Create fake locations where all actors are scheduled to the same parallel unit.
+            // Create fake locations where all actors are scheduled to the same worker.
             let locations = {
-                let StreamingClusterInfo {
-                    worker_nodes,
-                    parallel_units,
-                    unschedulable_parallel_units: _,
-                }: StreamingClusterInfo = self
+                let StreamingClusterInfo { worker_nodes, .. }: StreamingClusterInfo = self
                     .global_stream_manager
                     .metadata_manager
                     .get_streaming_cluster_info()
                     .await?;
 
+                let (worker_id, _worker_node) = worker_nodes.iter().exactly_one().unwrap();
+
                 let actor_locations = fragments
                     .values()
                     .flat_map(|f| &f.actors)
                     .sorted_by(|a, b| a.actor_id.cmp(&b.actor_id))
                     .enumerate()
-                    .map(|(idx, a)| (a.actor_id, parallel_units[&(idx as u32)].clone()))
+                    .map(|(idx, a)| (a.actor_id, WorkerSlotId::new(*worker_id, idx)))
                     .collect();
 
                 Locations {
@@ -1124,7 +1166,17 @@ mod tests {
             );
             let ctx = CreateStreamingJobContext {
                 building_locations: locations,
-                ..Default::default()
+                streaming_job: StreamingJob::MaterializedView(table.clone()),
+                mv_table_id: Some(table_fragments.table_id().table_id),
+                dispatchers: Default::default(),
+                upstream_root_actors: Default::default(),
+                internal_tables: Default::default(),
+                existing_locations: Default::default(),
+                definition: "".to_string(),
+                create_type: Default::default(),
+                ddl_type: Default::default(),
+                replace_table_job_info: None,
+                option: Default::default(),
             };
 
             self.catalog_manager
@@ -1167,9 +1219,17 @@ mod tests {
     }
 
     fn make_mview_stream_actors(table_id: &TableId, count: usize) -> Vec<StreamActor> {
+        let mut actor_bitmaps: HashMap<_, _> =
+            ActorMapping::new_uniform((0..count).map(|i| i as hash::ActorId))
+                .to_bitmaps()
+                .into_iter()
+                .map(|(actor_id, bitmap)| (actor_id, bitmap.to_protobuf()))
+                .collect();
+
         (0..count)
             .map(|i| StreamActor {
                 actor_id: i as u32,
+                vnode_bitmap: actor_bitmaps.remove(&(i as u32)),
                 // A dummy node to avoid panic.
                 nodes: Some(StreamNode {
                     node_body: Some(NodeBody::Materialize(MaterializeNode {
@@ -1191,15 +1251,8 @@ mod tests {
         let table_id = TableId::new(0);
         let actors = make_mview_stream_actors(&table_id, 4);
 
-        let StreamingClusterInfo { parallel_units, .. } = services
-            .global_stream_manager
-            .metadata_manager
-            .get_streaming_cluster_info()
-            .await?;
-
-        let parallel_unit_ids = parallel_units.keys().cloned().sorted().collect_vec();
-
         let mut fragments = BTreeMap::default();
+
         fragments.insert(
             0,
             Fragment {
@@ -1208,9 +1261,6 @@ mod tests {
                 distribution_type: FragmentDistributionType::Hash as i32,
                 actors: actors.clone(),
                 state_table_ids: vec![0],
-                vnode_mapping: Some(
-                    ParallelUnitMapping::new_uniform(parallel_unit_ids.into_iter()).to_protobuf(),
-                ),
                 ..Default::default()
             },
         );
@@ -1268,7 +1318,6 @@ mod tests {
                 distribution_type: FragmentDistributionType::Hash as i32,
                 actors: actors.clone(),
                 state_table_ids: vec![0],
-                vnode_mapping: Some(ParallelUnitMapping::new_single(0).to_protobuf()),
                 ..Default::default()
             },
         );
diff --git a/src/meta/src/stream/test_fragmenter.rs b/src/meta/src/stream/test_fragmenter.rs
index 510011e69eb88..601c708efef34 100644
--- a/src/meta/src/stream/test_fragmenter.rs
+++ b/src/meta/src/stream/test_fragmenter.rs
@@ -19,9 +19,7 @@ use std::vec;
 use itertools::Itertools;
 use risingwave_common::catalog::{DatabaseId, SchemaId, TableId};
 use risingwave_pb::catalog::PbTable;
-use risingwave_pb::common::{
-    ParallelUnit, PbColumnOrder, PbDirection, PbNullsAre, PbOrderType, WorkerNode,
-};
+use risingwave_pb::common::{PbColumnOrder, PbDirection, PbNullsAre, PbOrderType, WorkerNode};
 use risingwave_pb::data::data_type::TypeName;
 use risingwave_pb::data::DataType;
 use risingwave_pb::ddl_service::TableJobType;
@@ -75,6 +73,7 @@ fn make_sum_aggcall(idx: u32) -> AggCall {
         filter: None,
         direct_args: vec![],
         udf: None,
+        scalar: None,
     }
 }
 
@@ -420,31 +419,18 @@ fn make_stream_graph() -> StreamFragmentGraphProto {
 }
 
 fn make_cluster_info() -> StreamingClusterInfo {
-    let parallel_units = (0..8)
-        .map(|id| {
-            (
-                id,
-                ParallelUnit {
-                    id,
-                    worker_node_id: 0,
-                },
-            )
-        })
-        .collect();
-
     let worker_nodes = std::iter::once((
         0,
         WorkerNode {
             id: 0,
+            parallelism: 8,
             ..Default::default()
         },
     ))
     .collect();
-    let unschedulable_parallel_units = Default::default();
     StreamingClusterInfo {
         worker_nodes,
-        parallel_units,
-        unschedulable_parallel_units,
+        unschedulable_workers: Default::default(),
     }
 }
 
diff --git a/src/meta/src/stream/test_scale.rs b/src/meta/src/stream/test_scale.rs
index 73d59ff52f2f4..0dc0bced84005 100644
--- a/src/meta/src/stream/test_scale.rs
+++ b/src/meta/src/stream/test_scale.rs
@@ -18,15 +18,14 @@ mod tests {
 
     use itertools::Itertools;
     use maplit::btreeset;
-    use risingwave_common::buffer::Bitmap;
-    use risingwave_common::hash::{ActorMapping, ParallelUnitId, ParallelUnitMapping, VirtualNode};
-    use risingwave_pb::common::ParallelUnit;
+    use risingwave_common::bitmap::Bitmap;
+    use risingwave_common::hash::{ActorMapping, VirtualNode};
 
     use crate::model::ActorId;
     use crate::stream::scale::rebalance_actor_vnode;
     use crate::stream::CustomActorInfo;
 
-    fn simulated_parallel_unit_nums(min: Option<usize>, max: Option<usize>) -> Vec<usize> {
+    fn simulated_parallelism(min: Option<usize>, max: Option<usize>) -> Vec<usize> {
         let mut raw = vec![1, 3, 12, 42, VirtualNode::COUNT];
         if let Some(min) = min {
             raw.retain(|n| *n > min);
@@ -39,31 +38,20 @@ mod tests {
         raw
     }
 
-    fn build_fake_actors(info: &[(ActorId, ParallelUnitId)]) -> Vec<CustomActorInfo> {
-        let parallel_units = generate_parallel_units(info);
-
-        let vnode_bitmaps = ParallelUnitMapping::build(&parallel_units).to_bitmaps();
-
-        info.iter()
-            .map(|(actor_id, parallel_unit_id)| CustomActorInfo {
+    fn build_fake_actors(actor_ids: Vec<ActorId>) -> Vec<CustomActorInfo> {
+        let actor_bitmaps = ActorMapping::new_uniform(actor_ids.clone().into_iter()).to_bitmaps();
+        actor_ids
+            .iter()
+            .map(|actor_id| CustomActorInfo {
                 actor_id: *actor_id,
-                vnode_bitmap: vnode_bitmaps
-                    .get(parallel_unit_id)
+                vnode_bitmap: actor_bitmaps
+                    .get(actor_id)
                     .map(|bitmap| bitmap.to_protobuf()),
                 ..Default::default()
             })
             .collect()
     }
 
-    fn generate_parallel_units(info: &[(ActorId, ParallelUnitId)]) -> Vec<ParallelUnit> {
-        info.iter()
-            .map(|(_, parallel_unit_id)| ParallelUnit {
-                id: *parallel_unit_id,
-                ..Default::default()
-            })
-            .collect_vec()
-    }
-
     fn check_affinity_for_scale_in(bitmap: &Bitmap, actor: &CustomActorInfo) {
         let prev_bitmap = Bitmap::from(actor.vnode_bitmap.as_ref().unwrap());
 
@@ -98,22 +86,19 @@ mod tests {
     }
 
     #[test]
-    fn test_build_vnode_mapping() {
-        for parallel_units_num in simulated_parallel_unit_nums(None, None) {
-            let info = (0..parallel_units_num)
-                .map(|i| (i as ActorId, i as ParallelUnitId))
-                .collect_vec();
-            let parallel_units = generate_parallel_units(&info);
-            let vnode_mapping = ParallelUnitMapping::build(&parallel_units);
+    fn test_build_actor_mapping() {
+        for parallelism in simulated_parallelism(None, None) {
+            let actor_ids = (0..parallelism as ActorId).collect_vec();
+            let actor_mapping = ActorMapping::new_uniform(actor_ids.into_iter());
 
-            assert_eq!(vnode_mapping.len(), VirtualNode::COUNT);
+            assert_eq!(actor_mapping.len(), VirtualNode::COUNT);
 
             let mut check: HashMap<u32, Vec<_>> = HashMap::new();
-            for (vnode, parallel_unit_id) in vnode_mapping.iter_with_vnode() {
-                check.entry(parallel_unit_id).or_default().push(vnode);
+            for (vnode, actor_id) in actor_mapping.iter_with_vnode() {
+                check.entry(actor_id).or_default().push(vnode);
             }
 
-            assert_eq!(check.len(), parallel_units_num);
+            assert_eq!(check.len(), parallelism);
 
             let (min, max) = check
                 .values()
@@ -126,46 +111,9 @@ mod tests {
         }
     }
 
-    #[test]
-    fn test_vnode_mapping_to_bitmaps() {
-        for parallel_units_num in simulated_parallel_unit_nums(None, None) {
-            let info = (0..parallel_units_num)
-                .map(|i| (i as ActorId, i as ParallelUnitId))
-                .collect_vec();
-            let parallel_units = generate_parallel_units(&info);
-            let bitmaps = ParallelUnitMapping::build(&parallel_units).to_bitmaps();
-            check_bitmaps(&bitmaps);
-        }
-    }
-
-    #[test]
-    fn test_mapping_convert() {
-        for parallel_unit_num in simulated_parallel_unit_nums(None, None) {
-            let (actor_mapping, _) = generate_actor_mapping(parallel_unit_num);
-
-            let actor_to_parallel_unit_map: HashMap<_, _> = (0..parallel_unit_num)
-                .map(|i| (i as ActorId, i as ParallelUnitId))
-                .collect();
-            let parallel_unit_mapping = actor_mapping.to_parallel_unit(&actor_to_parallel_unit_map);
-
-            let parallel_unit_to_actor_map: HashMap<_, _> = actor_to_parallel_unit_map
-                .into_iter()
-                .map(|(k, v)| (v, k))
-                .collect();
-
-            let new_actor_mapping = parallel_unit_mapping.to_actor(&parallel_unit_to_actor_map);
-
-            assert_eq!(actor_mapping, new_actor_mapping)
-        }
-    }
-
-    fn generate_actor_mapping(
-        parallel_unit_num: usize,
-    ) -> (ActorMapping, HashMap<ActorId, Bitmap>) {
-        let parallel_units = (0..parallel_unit_num)
-            .map(|i| (i as ActorId, i as ParallelUnitId))
-            .collect_vec();
-        let actors = build_fake_actors(&parallel_units);
+    fn generate_actor_mapping(parallelism: usize) -> (ActorMapping, HashMap<ActorId, Bitmap>) {
+        let actor_ids = (0..parallelism).map(|i| i as ActorId).collect_vec();
+        let actors = build_fake_actors(actor_ids);
 
         let bitmaps: HashMap<_, _> = actors
             .into_iter()
@@ -182,8 +130,8 @@ mod tests {
 
     #[test]
     fn test_actor_mapping_from_bitmaps() {
-        for parallel_unit_num in simulated_parallel_unit_nums(None, None) {
-            let (actor_mapping, bitmaps) = generate_actor_mapping(parallel_unit_num);
+        for parallelism in simulated_parallelism(None, None) {
+            let (actor_mapping, bitmaps) = generate_actor_mapping(parallelism);
             check_bitmaps(&bitmaps);
 
             for (actor_id, bitmap) in &bitmaps {
@@ -198,7 +146,7 @@ mod tests {
 
     #[test]
     fn test_rebalance_empty() {
-        let actors = build_fake_actors(&(0..3).map(|i| (i, i)).collect_vec());
+        let actors = build_fake_actors((0..3 as ActorId).collect_vec());
 
         // empty input
         let result = rebalance_actor_vnode(&actors, &BTreeSet::new(), &BTreeSet::new());
@@ -207,12 +155,8 @@ mod tests {
 
     #[test]
     fn test_rebalance_scale_in() {
-        for parallel_unit_num in simulated_parallel_unit_nums(Some(3), None) {
-            let actors = build_fake_actors(
-                &(0..parallel_unit_num)
-                    .map(|i| (i as ActorId, i as ParallelUnitId))
-                    .collect_vec(),
-            );
+        for parallelism in simulated_parallelism(Some(3), None) {
+            let actors = build_fake_actors((0..parallelism as ActorId).collect_vec());
 
             // remove 1
             let actors_to_remove = btreeset! {0};
@@ -222,7 +166,7 @@ mod tests {
             check_affinity_for_scale_in(result.get(&(1 as ActorId)).unwrap(), &actors[1]);
 
             // remove n-1
-            let actors_to_remove = (1..parallel_unit_num as ActorId).collect();
+            let actors_to_remove = (1..parallelism as ActorId).collect();
             let result = rebalance_actor_vnode(&actors, &actors_to_remove, &BTreeSet::new());
             assert_eq!(result.len(), 1);
             check_bitmaps(&result);
@@ -234,28 +178,19 @@ mod tests {
 
     #[test]
     fn test_rebalance_scale_out() {
-        for parallel_unit_num in simulated_parallel_unit_nums(Some(3), Some(VirtualNode::COUNT - 1))
-        {
-            let actors = build_fake_actors(
-                &(0..parallel_unit_num)
-                    .map(|i| (i as ActorId, i as ParallelUnitId))
-                    .collect_vec(),
-            );
+        for parallelism in simulated_parallelism(Some(3), Some(VirtualNode::COUNT - 1)) {
+            let actors = build_fake_actors((0..parallelism as ActorId).collect_vec());
 
             // add 1
-            let actors_to_add = btreeset! {parallel_unit_num as ActorId};
+            let actors_to_add = btreeset! {parallelism as ActorId};
             let result = rebalance_actor_vnode(&actors, &BTreeSet::new(), &actors_to_add);
             assert_eq!(result.len(), actors.len() + actors_to_add.len());
             check_bitmaps(&result);
 
-            let actors = build_fake_actors(
-                &(0..parallel_unit_num)
-                    .map(|i| (i as ActorId, i as ParallelUnitId))
-                    .collect_vec(),
-            );
+            let actors = build_fake_actors((0..parallelism as ActorId).collect_vec());
+
             // add to VirtualNode::COUNT
-            let actors_to_add =
-                (parallel_unit_num as ActorId..VirtualNode::COUNT as ActorId).collect();
+            let actors_to_add = (parallelism as ActorId..VirtualNode::COUNT as ActorId).collect();
             let result = rebalance_actor_vnode(&actors, &BTreeSet::new(), &actors_to_add);
             assert_eq!(result.len(), actors.len() + actors_to_add.len());
             check_bitmaps(&result);
@@ -264,16 +199,12 @@ mod tests {
 
     #[test]
     fn test_rebalance_migration() {
-        for parallel_unit_num in simulated_parallel_unit_nums(Some(3), None) {
-            let actors = build_fake_actors(
-                &(0..parallel_unit_num)
-                    .map(|i| (i as ActorId, i as ParallelUnitId))
-                    .collect_vec(),
-            );
+        for parallelism in simulated_parallelism(Some(3), None) {
+            let actors = build_fake_actors((0..parallelism as ActorId).collect_vec());
 
-            for idx in 0..parallel_unit_num {
+            for idx in 0..parallelism {
                 let actors_to_remove = btreeset! {idx as ActorId};
-                let actors_to_add = btreeset! {parallel_unit_num as ActorId};
+                let actors_to_add = btreeset! {parallelism as ActorId};
                 let result = rebalance_actor_vnode(&actors, &actors_to_remove, &actors_to_add);
 
                 assert_eq!(
@@ -293,18 +224,12 @@ mod tests {
                     assert!(prev_bitmap.eq(target_bitmap));
                 }
             }
+            let actors = build_fake_actors((0..parallelism as ActorId).collect_vec());
 
-            let actors = build_fake_actors(
-                &(0..parallel_unit_num)
-                    .map(|i| (i as ActorId, i as ParallelUnitId))
-                    .collect_vec(),
-            );
-
-            for migration_count in 1..parallel_unit_num {
+            for migration_count in 1..parallelism {
                 let actors_to_remove = (0..migration_count as ActorId).collect();
-                let actors_to_add = (parallel_unit_num as ActorId
-                    ..(parallel_unit_num + migration_count) as ActorId)
-                    .collect();
+                let actors_to_add =
+                    (parallelism as ActorId..(parallelism + migration_count) as ActorId).collect();
                 let result = rebalance_actor_vnode(&actors, &actors_to_remove, &actors_to_add);
 
                 assert_eq!(
@@ -319,16 +244,13 @@ mod tests {
 
     #[test]
     fn test_rebalance_scale() {
-        for parallel_unit_num in simulated_parallel_unit_nums(Some(3), None) {
-            let actors = build_fake_actors(
-                &(0..parallel_unit_num)
-                    .map(|i| (i as ActorId, i as ParallelUnitId))
-                    .collect_vec(),
-            );
+        for parallelism in simulated_parallelism(Some(3), None) {
+            let actor_ids = (0..parallelism as ActorId).collect_vec();
+            let actors = build_fake_actors(actor_ids);
 
-            let parallel_unit_num = parallel_unit_num as ActorId;
+            let parallelism = parallelism as ActorId;
             let actors_to_remove = btreeset! {0};
-            let actors_to_add = btreeset! {parallel_unit_num, parallel_unit_num+1};
+            let actors_to_add = btreeset! {parallelism, parallelism+1};
             let result = rebalance_actor_vnode(&actors, &actors_to_remove, &actors_to_add);
 
             assert_eq!(
@@ -338,7 +260,7 @@ mod tests {
             check_bitmaps(&result);
 
             let actors_to_remove = btreeset! {0, 1};
-            let actors_to_add = btreeset! {parallel_unit_num};
+            let actors_to_add = btreeset! {parallelism};
             let result = rebalance_actor_vnode(&actors, &actors_to_remove, &actors_to_add);
 
             assert_eq!(
@@ -353,11 +275,8 @@ mod tests {
 
     #[test]
     fn test_rebalance_scale_real() {
-        let parallel_units = (0..(VirtualNode::COUNT - 1) as ActorId)
-            .map(|i| (i, i))
-            .collect_vec();
-        let actors = build_fake_actors(&parallel_units);
-
+        let actor_ids = (0..(VirtualNode::COUNT - 1) as ActorId).collect_vec();
+        let actors = build_fake_actors(actor_ids);
         let actors_to_remove = btreeset! {0, 1};
         let actors_to_add = btreeset! {255};
         let result = rebalance_actor_vnode(&actors, &actors_to_remove, &actors_to_add);
diff --git a/src/meta/src/telemetry.rs b/src/meta/src/telemetry.rs
index 27456289aa3d3..a76d76c58c6e7 100644
--- a/src/meta/src/telemetry.rs
+++ b/src/meta/src/telemetry.rs
@@ -15,14 +15,18 @@
 use prost::Message;
 use risingwave_common::config::MetaBackend;
 use risingwave_common::telemetry::pb_compatible::TelemetryToProtobuf;
-use risingwave_common::telemetry::report::{TelemetryInfoFetcher, TelemetryReportCreator};
+use risingwave_common::telemetry::report::{
+    report_event_common, TelemetryInfoFetcher, TelemetryReportCreator,
+};
 use risingwave_common::telemetry::{
     current_timestamp, telemetry_cluster_type_from_env_var, SystemData, TelemetryNodeType,
     TelemetryReportBase, TelemetryResult,
 };
 use risingwave_common::{GIT_SHA, RW_VERSION};
 use risingwave_pb::common::WorkerType;
-use risingwave_pb::telemetry::PbTelemetryClusterType;
+use risingwave_pb::telemetry::{
+    PbTelemetryClusterType, PbTelemetryDatabaseObject, PbTelemetryEventStage,
+};
 use serde::{Deserialize, Serialize};
 use thiserror_ext::AsReport;
 
@@ -31,6 +35,25 @@ use crate::model::ClusterId;
 
 const TELEMETRY_META_REPORT_TYPE: &str = "meta";
 
+pub(crate) fn report_event(
+    event_stage: PbTelemetryEventStage,
+    event_name: &str,
+    catalog_id: i64,
+    connector_name: Option<String>,
+    component: Option<PbTelemetryDatabaseObject>,
+    attributes: Option<jsonbb::Value>, // any json string
+) {
+    report_event_common(
+        event_stage,
+        event_name,
+        catalog_id,
+        connector_name,
+        component,
+        attributes,
+        TELEMETRY_META_REPORT_TYPE.to_string(),
+    );
+}
+
 #[derive(Debug, Serialize, Deserialize)]
 struct NodeCount {
     meta_count: u64,
diff --git a/src/object_store/Cargo.toml b/src/object_store/Cargo.toml
index 38bef42305792..e821c2fc85090 100644
--- a/src/object_store/Cargo.toml
+++ b/src/object_store/Cargo.toml
@@ -31,7 +31,7 @@ hyper-rustls = { version = "0.24.2", features = ["webpki-roots"] }
 hyper-tls = "0.5.0"
 itertools = { workspace = true }
 madsim = "0.2.27"
-opendal = { version = "0.47", features = [
+opendal = { workspace = true, features = [
     "executors-tokio",
     "services-azblob",
     "services-fs",
@@ -42,7 +42,6 @@ opendal = { version = "0.47", features = [
     "services-s3",
     "services-webhdfs",
     "services-azfile",
-    # "service-hdfs",
 ] }
 prometheus = { version = "0.13", features = ["process"] }
 reqwest = "0.12.2" # required by opendal
diff --git a/src/object_store/src/lib.rs b/src/object_store/src/lib.rs
index 811d482587acb..d9e768b7f0290 100644
--- a/src/object_store/src/lib.rs
+++ b/src/object_store/src/lib.rs
@@ -14,7 +14,6 @@
 
 #![feature(trait_alias)]
 #![feature(type_alias_impl_trait)]
-#![feature(lazy_cell)]
 #![feature(lint_reasons)]
 #![feature(error_generic_member_access)]
 #![feature(let_chains)]
diff --git a/src/object_store/src/object/mod.rs b/src/object_store/src/object/mod.rs
index 46139804c6576..8ee8dc078fe17 100644
--- a/src/object_store/src/object/mod.rs
+++ b/src/object_store/src/object/mod.rs
@@ -12,6 +12,11 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#![expect(
+    unexpected_cfgs,
+    reason = "feature(hdfs-backend) is banned https://github.com/risingwavelabs/risingwave/pull/7875"
+)]
+
 pub mod sim;
 use std::ops::{Range, RangeBounds};
 use std::sync::Arc;
@@ -390,12 +395,11 @@ impl<U: StreamingUploader> MonitoredStreamingUploader<U> {
         let operation_type_str = operation_type.as_str();
         let data_len = data.len();
 
-        let res =
-            // TODO: we should avoid this special case after fully migrating to opeandal for s3.
-            self.inner
-                .write_bytes(data)
-                .verbose_instrument_await(operation_type_str)
-                .await;
+        let res = self
+            .inner
+            .write_bytes(data)
+            .verbose_instrument_await(operation_type_str)
+            .await;
 
         try_update_failure_metric(&self.object_store_metrics, &res, operation_type_str);
 
diff --git a/src/object_store/src/object/opendal_engine/hdfs.rs b/src/object_store/src/object/opendal_engine/hdfs.rs
index 093a9f2d6c658..b7b28ef08a054 100644
--- a/src/object_store/src/object/opendal_engine/hdfs.rs
+++ b/src/object_store/src/object/opendal_engine/hdfs.rs
@@ -20,7 +20,7 @@ use opendal::Operator;
 use risingwave_common::config::ObjectStoreConfig;
 
 use super::{EngineType, OpendalObjectStore};
-use crate::object::opendal_engine::ATOMIC_WRITE_DIR;
+// use crate::object::opendal_engine::ATOMIC_WRITE_DIR;
 use crate::object::ObjectResult;
 
 impl OpendalObjectStore {
@@ -43,10 +43,11 @@ impl OpendalObjectStore {
         // Set the name node for hdfs.
         builder.name_node(&namenode);
         builder.root(&root);
-        if config.set_atomic_write_dir {
-            let atomic_write_dir = format!("{}/{}", root, ATOMIC_WRITE_DIR);
-            builder.atomic_write_dir(&atomic_write_dir);
-        }
+        // todo: reopen the following lines after https://github.com/apache/opendal/issues/4867 is resolved.
+        // if config.set_atomic_write_dir {
+        //     let atomic_write_dir = format!("{}/{}", root, ATOMIC_WRITE_DIR);
+        //     builder.atomic_write_dir(&atomic_write_dir);
+        // }
         let op: Operator = Operator::new(builder)?
             .layer(LoggingLayer::default())
             .finish();
diff --git a/src/object_store/src/object/opendal_engine/opendal_s3.rs b/src/object_store/src/object/opendal_engine/opendal_s3.rs
index 5ba90ad93ccba..183496d08673a 100644
--- a/src/object_store/src/object/opendal_engine/opendal_s3.rs
+++ b/src/object_store/src/object/opendal_engine/opendal_s3.rs
@@ -135,6 +135,7 @@ impl OpendalObjectStore {
         builder.access_key_id(aws_access_key_id);
         builder.secret_access_key(aws_secret_access_key);
         builder.region(aws_region);
+        builder.disable_config_load();
 
         let http_client = Self::new_http_client(config.as_ref())?;
         builder.http_client(http_client);
diff --git a/src/object_store/src/object/s3.rs b/src/object_store/src/object/s3.rs
index 3ed5fc01ba40c..001eb8128a5b2 100644
--- a/src/object_store/src/object/s3.rs
+++ b/src/object_store/src/object/s3.rs
@@ -53,7 +53,7 @@ use futures::{stream, Stream, StreamExt, TryStreamExt};
 use hyper::Body;
 use itertools::Itertools;
 use risingwave_common::config::ObjectStoreConfig;
-use risingwave_common::monitor::connection::monitor_connector;
+use risingwave_common::monitor::monitor_connector;
 use risingwave_common::range::RangeBoundsExt;
 use thiserror_ext::AsReport;
 use tokio::task::JoinHandle;
@@ -71,13 +71,6 @@ type PartId = i32;
 
 /// MinIO and S3 share the same minimum part ID and part size.
 const MIN_PART_ID: PartId = 1;
-/// The minimum number of bytes that is buffered before they are uploaded as a part.
-/// Its value must be greater than the minimum part size of 5MiB.
-///
-/// Reference: <https://docs.aws.amazon.com/AmazonS3/latest/userguide/qfacts.html>
-const S3_PART_SIZE: usize = 16 * 1024 * 1024;
-// TODO: we should do some benchmark to determine the proper part size for MinIO
-const MINIO_PART_SIZE: usize = 16 * 1024 * 1024;
 /// Stop multipart uploads that don't complete within a specified number of days after being
 /// initiated. (Day is the smallest granularity)
 const S3_INCOMPLETE_MULTIPART_UPLOAD_RETENTION_DAYS: i32 = 1;
@@ -113,11 +106,18 @@ impl S3StreamingUploader {
     pub fn new(
         client: Client,
         bucket: String,
-        part_size: usize,
         key: String,
         metrics: Arc<ObjectStoreMetrics>,
         config: Arc<ObjectStoreConfig>,
     ) -> S3StreamingUploader {
+        /// The minimum number of bytes that is buffered before they are uploaded as a part.
+        /// Its value must be greater than the minimum part size of 5MiB.
+        ///
+        /// Reference: <https://docs.aws.amazon.com/AmazonS3/latest/userguide/qfacts.html>
+        const MIN_PART_SIZE: usize = 5 * 1024 * 1024;
+        const MAX_PART_SIZE: usize = 5 * 1024 * 1024 * 1024;
+        let part_size = config.upload_part_size.clamp(MIN_PART_SIZE, MAX_PART_SIZE);
+
         Self {
             client,
             bucket,
@@ -391,7 +391,6 @@ fn get_upload_body(data: Vec<Bytes>) -> ByteStream {
 pub struct S3ObjectStore {
     client: Client,
     bucket: String,
-    part_size: usize,
     /// For S3 specific metrics.
     metrics: Arc<ObjectStoreMetrics>,
 
@@ -436,7 +435,6 @@ impl ObjectStore for S3ObjectStore {
         Ok(S3StreamingUploader::new(
             self.client.clone(),
             self.bucket.clone(),
-            self.part_size,
             path.to_string(),
             self.metrics.clone(),
             self.config.clone(),
@@ -715,7 +713,6 @@ impl S3ObjectStore {
         Self {
             client,
             bucket,
-            part_size: S3_PART_SIZE,
             metrics,
             config,
         }
@@ -778,7 +775,6 @@ impl S3ObjectStore {
         Self {
             client,
             bucket: bucket.to_string(),
-            part_size: MINIO_PART_SIZE,
             metrics,
             config: object_store_config,
         }
diff --git a/src/prost/build.rs b/src/prost/build.rs
index 6d31201fa4733..f8e651e7103a3 100644
--- a/src/prost/build.rs
+++ b/src/prost/build.rs
@@ -66,10 +66,11 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
         ".plan_common.ExternalTableDesc",
         ".hummock.CompactTask",
         ".catalog.StreamSourceInfo",
-        ".catalog.SecretRef",
+        ".secret.SecretRef",
         ".catalog.Source",
         ".catalog.Sink",
         ".catalog.View",
+        ".catalog.SinkFormatDesc",
         ".connector_service.ValidateSourceRequest",
         ".connector_service.GetEventStreamRequest",
         ".connector_service.SinkParam",
@@ -119,6 +120,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
         .type_attribute("expr.ExprNode.rex_node", "#[derive(Eq, Hash)]")
         .type_attribute("expr.ExprNode.NowRexNode", "#[derive(Eq, Hash)]")
         .type_attribute("expr.InputRef", "#[derive(Eq, Hash)]")
+        .type_attribute("expr.UserDefinedFunctionMetadata", "#[derive(Eq, Hash)]")
         .type_attribute("data.Datum", "#[derive(Eq, Hash)]")
         .type_attribute("expr.FunctionCall", "#[derive(Eq, Hash)]")
         .type_attribute("expr.UserDefinedFunction", "#[derive(Eq, Hash)]")
diff --git a/src/prost/src/lib.rs b/src/prost/src/lib.rs
index 27d0523b84115..0f02033c5f003 100644
--- a/src/prost/src/lib.rs
+++ b/src/prost/src/lib.rs
@@ -184,6 +184,14 @@ impl FromStr for crate::expr::table_function::PbType {
     }
 }
 
+impl FromStr for crate::expr::agg_call::PbType {
+    type Err = ();
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        Self::from_str_name(&s.to_uppercase()).ok_or(())
+    }
+}
+
 impl stream_plan::MaterializeNode {
     pub fn dist_key_indices(&self) -> Vec<u32> {
         self.get_table()
@@ -204,6 +212,13 @@ impl stream_plan::MaterializeNode {
     }
 }
 
+// Encapsulating the use of parallelism.
+impl common::WorkerNode {
+    pub fn parallelism(&self) -> usize {
+        self.parallelism as usize
+    }
+}
+
 impl stream_plan::SourceNode {
     pub fn column_ids(&self) -> Option<Vec<i32>> {
         Some(
@@ -217,6 +232,21 @@ impl stream_plan::SourceNode {
     }
 }
 
+impl meta::table_fragments::ActorStatus {
+    pub fn worker_id(&self) -> u32 {
+        self.location
+            .as_ref()
+            .expect("actor location should be exist")
+            .worker_node_id
+    }
+}
+
+impl common::ActorLocation {
+    pub fn from_worker(worker_node_id: u32) -> Option<Self> {
+        Some(Self { worker_node_id })
+    }
+}
+
 impl stream_plan::StreamNode {
     /// Find the external stream source info inside the stream node, if any.
     ///
@@ -266,6 +296,17 @@ impl catalog::StreamSourceInfo {
     }
 }
 
+impl catalog::Sink {
+    // TODO: remove this placeholder
+    // creating table sink does not have an id, so we need a placeholder
+    pub const UNIQUE_IDENTITY_FOR_CREATING_TABLE_SINK: &'static str = "PLACE_HOLDER";
+
+    pub fn unique_identity(&self) -> String {
+        // TODO: use a more unique name
+        format!("{}", self.id)
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use crate::data::{data_type, DataType};
diff --git a/src/risedevtool/common.toml b/src/risedevtool/common.toml
index 391d52f399cf9..960ede84de140 100644
--- a/src/risedevtool/common.toml
+++ b/src/risedevtool/common.toml
@@ -1,4 +1,5 @@
 [env]
+CARGO_MAKE_CRATE_INSTALLATION_LOCKED = true
 RISEDEV = "1"
 RUST_BACKTRACE = "1"
 OS = { source = "${CARGO_MAKE_RUST_TARGET_OS}", mapping = { linux = "linux", macos = "darwin" } }
@@ -15,6 +16,7 @@ PREFIX_LOG = "${PREFIX}/log"
 PREFIX_TMP = "${PREFIX}/tmp"
 PREFIX_DOCKER = "${PREFIX}/rw-docker"
 PREFIX_PROFILING = "${PREFIX}/profiling"
+PREFIX_SECRET = "${PREFIX}/secrets"
 
 BUILD_MODE_DIR = { source = "${ENABLE_RELEASE_PROFILE}", default_value = "debug", mapping = { true = "release" } }
 RISINGWAVE_BUILD_PROFILE = { source = "${ENABLE_RELEASE_PROFILE}", default_value = "dev", mapping = { true = "release" } }
diff --git a/src/risedevtool/config/src/main.rs b/src/risedevtool/config/src/main.rs
index 27a676c2bfb3e..fda54b82088d9 100644
--- a/src/risedevtool/config/src/main.rs
+++ b/src/risedevtool/config/src/main.rs
@@ -77,7 +77,6 @@ pub enum Components {
     ExternalUdf,
     WasmUdf,
     JsUdf,
-    DenoUdf,
     PythonUdf,
 }
 
@@ -103,7 +102,6 @@ impl Components {
             Self::ExternalUdf => "[Build] Enable external UDF",
             Self::WasmUdf => "[Build] Enable Wasm UDF",
             Self::JsUdf => "[Build] Enable JS UDF",
-            Self::DenoUdf => "[Build] Enable Deno UDF",
             Self::PythonUdf => "[Build] Enable Python UDF",
         }
         .into()
@@ -205,7 +203,6 @@ With this option enabled, RiseDev will not set `RUST_BACKTRACE` when launching n
             Self::ExternalUdf => "Required if you want to support external UDF.",
             Self::WasmUdf => "Required if you want to support WASM UDF.",
             Self::JsUdf => "Required if you want to support JS UDF.",
-            Self::DenoUdf => "Required if you want to support Deno UDF.",
             Self::PythonUdf => "Required if you want to support Python UDF.",
         }
         .into()
@@ -232,7 +229,6 @@ With this option enabled, RiseDev will not set `RUST_BACKTRACE` when launching n
             "ENABLE_EXTERNAL_UDF" => Some(Self::ExternalUdf),
             "ENABLE_WASM_UDF" => Some(Self::WasmUdf),
             "ENABLE_JS_UDF" => Some(Self::JsUdf),
-            "ENABLE_DENO_UDF" => Some(Self::DenoUdf),
             "ENABLE_PYTHON_UDF" => Some(Self::PythonUdf),
             _ => None,
         }
@@ -259,7 +255,6 @@ With this option enabled, RiseDev will not set `RUST_BACKTRACE` when launching n
             Self::ExternalUdf => "ENABLE_EXTERNAL_UDF",
             Self::WasmUdf => "ENABLE_WASM_UDF",
             Self::JsUdf => "ENABLE_JS_UDF",
-            Self::DenoUdf => "ENABLE_DENO_UDF",
             Self::PythonUdf => "ENABLE_PYTHON_UDF",
         }
         .into()
diff --git a/src/risedevtool/run_command.sh b/src/risedevtool/run_command.sh
index e3da246d9d8c9..02d58cf15217e 100755
--- a/src/risedevtool/run_command.sh
+++ b/src/risedevtool/run_command.sh
@@ -5,14 +5,32 @@
 echo "${@:3}"
 echo "logging to $1, and status to $2"
 
-"${@:3}" 2>&1 | tee "$1"
+# Strip ANSI color codes to make the log file readable.
+# - trap '' INT: ignore interrupts in sed
+# -      sed -u: unbuffered output
+strip_ansi() {
+  (trap '' INT; sed -u -e 's/\x1b\[[0-9;]*m//g')
+}
 
+# Run the command and log the output to both the terminal and the log file.
+# - CLICOLOR_FORCE=1: force color output, see https://bixense.com/clicolors/
+# -           tee -i: ignore interrupts in tee
+CLICOLOR_FORCE=1 "${@:3}" 2>&1 | tee -i >(strip_ansi > "$1")
+
+# Retrieve the return status.
 RET_STATUS="${PIPESTATUS[0]}"
 
-echo "status ${RET_STATUS}" > "$2"
+# If the status file exists, write the return status to it.
+#
+# The status file is used to detect early exits. Once `risedev-dev` finishes launching successfully,
+# it will clean up the status file, so it's safe to ignore it then.
+if [ -f "$2" ]; then
+    echo "status ${RET_STATUS}" > "$2"
+fi
 
+# Show the return status in both outputs.
 echo "$(date -u) [risedev]: Program exited with ${RET_STATUS}" >> "$1"
-echo "Program exited with ${RET_STATUS}, press Ctrl+C to continue."
+echo "Program exited with ${RET_STATUS}, press Ctrl+C again to close the window."
 
 while true; do
     read -r
diff --git a/src/risedevtool/src/bin/risedev-compose.rs b/src/risedevtool/src/bin/risedev-compose.rs
index 89bb0592d0d85..0547fda6b7008 100644
--- a/src/risedevtool/src/bin/risedev-compose.rs
+++ b/src/risedevtool/src/bin/risedev-compose.rs
@@ -222,6 +222,7 @@ fn main() -> Result<()> {
             ServiceConfig::Redis(_)
             | ServiceConfig::MySql(_)
             | ServiceConfig::Postgres(_)
+            | ServiceConfig::SqlServer(_)
             | ServiceConfig::SchemaRegistry(_) => return Err(anyhow!("not supported")),
         };
         compose.container_name = service.id().to_string();
diff --git a/src/risedevtool/src/bin/risedev-dev.rs b/src/risedevtool/src/bin/risedev-dev.rs
index 02cef8655db2f..c367473eb8e77 100644
--- a/src/risedevtool/src/bin/risedev-dev.rs
+++ b/src/risedevtool/src/bin/risedev-dev.rs
@@ -28,7 +28,7 @@ use risedev::{
     ConfigureTmuxTask, DummyService, EnsureStopService, ExecuteContext, FrontendService,
     GrafanaService, KafkaService, MetaNodeService, MinioService, MySqlService, PostgresService,
     PrometheusService, PubsubService, RedisService, SchemaRegistryService, ServiceConfig,
-    SqliteConfig, Task, TempoService, RISEDEV_NAME,
+    SqlServerService, SqliteConfig, Task, TempoService, RISEDEV_NAME,
 };
 use tempfile::tempdir;
 use thiserror_ext::AsReport;
@@ -284,9 +284,15 @@ fn task_main(
                     ExecuteContext::new(&mut logger, manager.new_progress(), status_dir.clone());
                 let mut service = SchemaRegistryService::new(c.clone());
                 service.execute(&mut ctx)?;
-                let mut task =
-                    risedev::TcpReadyCheckTask::new(c.address.clone(), c.port, c.user_managed)?;
-                task.execute(&mut ctx)?;
+                if c.user_managed {
+                    let mut task =
+                        risedev::TcpReadyCheckTask::new(c.address.clone(), c.port, c.user_managed)?;
+                    task.execute(&mut ctx)?;
+                } else {
+                    let mut task =
+                        risedev::LogReadyCheckTask::new("Server started, listening for requests")?;
+                    task.execute(&mut ctx)?;
+                }
                 ctx.pb
                     .set_message(format!("schema registry http://{}:{}", c.address, c.port));
             }
@@ -347,6 +353,24 @@ fn task_main(
                 ctx.pb
                     .set_message(format!("postgres {}:{}", c.address, c.port));
             }
+            ServiceConfig::SqlServer(c) => {
+                let mut ctx =
+                    ExecuteContext::new(&mut logger, manager.new_progress(), status_dir.clone());
+                // only `c.password` will be used in `SqlServerService` as the password for user `sa`.
+                SqlServerService::new(c.clone()).execute(&mut ctx)?;
+                if c.user_managed {
+                    let mut task =
+                        risedev::TcpReadyCheckTask::new(c.address.clone(), c.port, c.user_managed)?;
+                    task.execute(&mut ctx)?;
+                } else {
+                    let mut task = risedev::LogReadyCheckTask::new(
+                        "SQL Server is now ready for client connections.",
+                    )?;
+                    task.execute(&mut ctx)?;
+                }
+                ctx.pb
+                    .set_message(format!("sqlserver {}:{}", c.address, c.port));
+            }
         }
 
         let service_id = service.id().to_string();
diff --git a/src/risedevtool/src/config.rs b/src/risedevtool/src/config.rs
index bf768f8e68cd1..e4ba9acdf4e19 100644
--- a/src/risedevtool/src/config.rs
+++ b/src/risedevtool/src/config.rs
@@ -175,6 +175,7 @@ impl ConfigExpander {
                     "redpanda" => ServiceConfig::RedPanda(serde_yaml::from_str(&out_str)?),
                     "mysql" => ServiceConfig::MySql(serde_yaml::from_str(&out_str)?),
                     "postgres" => ServiceConfig::Postgres(serde_yaml::from_str(&out_str)?),
+                    "sqlserver" => ServiceConfig::SqlServer(serde_yaml::from_str(&out_str)?),
                     "schema-registry" => {
                         ServiceConfig::SchemaRegistry(serde_yaml::from_str(&out_str)?)
                     }
diff --git a/src/risedevtool/src/config/id_expander.rs b/src/risedevtool/src/config/id_expander.rs
index 01eae93843f5c..2c9f35cdee6ac 100644
--- a/src/risedevtool/src/config/id_expander.rs
+++ b/src/risedevtool/src/config/id_expander.rs
@@ -36,7 +36,7 @@ impl IdExpander {
                     .ok_or_else(|| anyhow!("Id isn't a string: {:?}", item))?;
                 ids.push(id.to_string());
             } else {
-                return Err(anyhow!("Not an hashmap: {:?}", item));
+                return Err(anyhow!("Not a hashmap: {:?}", item));
             }
         }
         Ok(Self { ids })
diff --git a/src/risedevtool/src/config_gen/grafana_gen.rs b/src/risedevtool/src/config_gen/grafana_gen.rs
index b326053e22737..1977be134fff6 100644
--- a/src/risedevtool/src/config_gen/grafana_gen.rs
+++ b/src/risedevtool/src/config_gen/grafana_gen.rs
@@ -50,7 +50,7 @@ org_role = Admin
         let provide_prometheus = config.provide_prometheus.as_ref().unwrap();
         if provide_prometheus.len() != 1 {
             return Err(anyhow!(
-                "expect 1 prometheus nodes, found {}",
+                "expect 1 prometheus node, found {}",
                 provide_prometheus.len()
             ));
         }
@@ -84,7 +84,7 @@ datasources:
         let provide_tempo = config.provide_tempo.as_ref().unwrap();
         if provide_tempo.len() != 1 {
             return Err(anyhow!(
-                "expect 1 tempo nodes, found {}",
+                "expect 1 tempo node, found {}",
                 provide_tempo.len()
             ));
         }
@@ -120,7 +120,7 @@ datasources:
         let provide_prometheus = config.provide_prometheus.as_ref().unwrap();
         if provide_prometheus.len() != 1 {
             return Err(anyhow!(
-                "expect 1 prometheus nodes, found {}",
+                "expect 1 prometheus node, found {}",
                 provide_prometheus.len()
             ));
         };
@@ -170,7 +170,7 @@ providers:
         let provide_prometheus = config.provide_prometheus.as_ref().unwrap();
         if provide_prometheus.len() != 1 {
             return Err(anyhow!(
-                "expect 1 prometheus nodes, found {}",
+                "expect 1 prometheus node, found {}",
                 provide_prometheus.len()
             ));
         };
diff --git a/src/risedevtool/src/risedev_env.rs b/src/risedevtool/src/risedev_env.rs
index bff4062f72097..4860d30e01485 100644
--- a/src/risedevtool/src/risedev_env.rs
+++ b/src/risedevtool/src/risedev_env.rs
@@ -110,11 +110,31 @@ pub fn generate_risedev_env(services: &Vec<ServiceConfig>) -> String {
                 let port = &c.port;
                 let user = &c.user;
                 let password = &c.password;
+                let database = &c.database;
                 // These envs are used by `postgres` cli.
                 writeln!(env, r#"PGHOST="{host}""#,).unwrap();
                 writeln!(env, r#"PGPORT="{port}""#,).unwrap();
                 writeln!(env, r#"PGUSER="{user}""#,).unwrap();
                 writeln!(env, r#"PGPASSWORD="{password}""#,).unwrap();
+                writeln!(env, r#"PGDATABASE="{database}""#,).unwrap();
+            }
+            ServiceConfig::SqlServer(c) => {
+                let host = &c.address;
+                let port = &c.port;
+                let user = &c.user;
+                let password = &c.password;
+                let database = &c.database;
+                // These envs are used by `sqlcmd`.
+                writeln!(env, r#"SQLCMDSERVER="{host}""#,).unwrap();
+                writeln!(env, r#"SQLCMDPORT="{port}""#,).unwrap();
+                writeln!(env, r#"SQLCMDUSER="{user}""#,).unwrap();
+                writeln!(env, r#"SQLCMDPASSWORD="{password}""#,).unwrap();
+                writeln!(env, r#"SQLCMDDBNAME="{database}""#,).unwrap();
+                writeln!(
+                    env,
+                    r#"RISEDEV_SQLSERVER_WITH_OPTIONS_COMMON="connector='sqlserver-cdc',hostname='{host}',port='{port}',username='{user}',password='{password}',database.name='{database}'""#,
+                )
+                .unwrap();
             }
             _ => {}
         }
diff --git a/src/risedevtool/src/service_config.rs b/src/risedevtool/src/service_config.rs
index fb95dbb520a0d..9d08a6f602251 100644
--- a/src/risedevtool/src/service_config.rs
+++ b/src/risedevtool/src/service_config.rs
@@ -412,6 +412,26 @@ pub struct PostgresConfig {
     pub persist_data: bool,
 }
 
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "kebab-case")]
+#[serde(deny_unknown_fields)]
+pub struct SqlServerConfig {
+    #[serde(rename = "use")]
+    phantom_use: Option<String>,
+    pub id: String,
+
+    pub port: u16,
+    pub address: String,
+
+    pub user: String,
+    pub password: String,
+    pub database: String,
+
+    pub image: String,
+    pub user_managed: bool,
+    pub persist_data: bool,
+}
+
 /// All service configuration
 #[derive(Clone, Debug, PartialEq)]
 pub enum ServiceConfig {
@@ -434,6 +454,7 @@ pub enum ServiceConfig {
     RedPanda(RedPandaConfig),
     MySql(MySqlConfig),
     Postgres(PostgresConfig),
+    SqlServer(SqlServerConfig),
 }
 
 impl ServiceConfig {
@@ -457,6 +478,7 @@ impl ServiceConfig {
             Self::Opendal(c) => &c.id,
             Self::MySql(c) => &c.id,
             Self::Postgres(c) => &c.id,
+            Self::SqlServer(c) => &c.id,
             Self::SchemaRegistry(c) => &c.id,
         }
     }
@@ -482,6 +504,7 @@ impl ServiceConfig {
             Self::Opendal(_) => None,
             Self::MySql(c) => Some(c.port),
             Self::Postgres(c) => Some(c.port),
+            Self::SqlServer(c) => Some(c.port),
             Self::SchemaRegistry(c) => Some(c.port),
         }
     }
@@ -506,6 +529,7 @@ impl ServiceConfig {
             Self::Opendal(_c) => false,
             Self::MySql(c) => c.user_managed,
             Self::Postgres(c) => c.user_managed,
+            Self::SqlServer(c) => c.user_managed,
             Self::SchemaRegistry(c) => c.user_managed,
         }
     }
diff --git a/src/risedevtool/src/task.rs b/src/risedevtool/src/task.rs
index 21b6f20eec5ee..dc82ede836cbd 100644
--- a/src/risedevtool/src/task.rs
+++ b/src/risedevtool/src/task.rs
@@ -30,6 +30,7 @@ mod prometheus_service;
 mod pubsub_service;
 mod redis_service;
 mod schema_registry_service;
+mod sql_server_service;
 mod task_configure_minio;
 mod task_etcd_ready_check;
 mod task_kafka_ready_check;
@@ -70,6 +71,7 @@ pub use self::prometheus_service::*;
 pub use self::pubsub_service::*;
 pub use self::redis_service::*;
 pub use self::schema_registry_service::SchemaRegistryService;
+pub use self::sql_server_service::*;
 pub use self::task_configure_minio::*;
 pub use self::task_etcd_ready_check::*;
 pub use self::task_kafka_ready_check::*;
@@ -138,11 +140,7 @@ where
         if !id.is_empty() {
             self.pb.set_prefix(id.clone());
             self.id = Some(id.clone());
-
-            // Remove the old status file if exists to avoid confusion.
-            let status_file = self.status_dir.path().join(format!("{}.status", id));
-            fs_err::remove_file(&status_file).ok();
-            self.status_file = Some(status_file);
+            self.status_file = Some(self.status_dir.path().join(format!("{}.status", id)));
 
             // Remove the old log file if exists to avoid confusion.
             let log_file = Path::new(&env::var("PREFIX_LOG").unwrap())
diff --git a/src/risedevtool/src/task/kafka_service.rs b/src/risedevtool/src/task/kafka_service.rs
index 7c415b6d9749a..24c6dff2e3e36 100644
--- a/src/risedevtool/src/task/kafka_service.rs
+++ b/src/risedevtool/src/task/kafka_service.rs
@@ -62,6 +62,11 @@ impl DockerServiceConfig for KafkaConfig {
                 "KAFKA_INTER_BROKER_LISTENER_NAME".to_owned(),
                 "HOST".to_owned(),
             ),
+            // https://docs.confluent.io/platform/current/installation/docker/config-reference.html#example-configurations
+            (
+                "KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR".to_owned(),
+                "1".to_owned(),
+            ),
             ("CLUSTER_ID".to_owned(), "RiseDevRiseDevRiseDev1".to_owned()),
         ]
     }
diff --git a/src/risedevtool/src/task/meta_node_service.rs b/src/risedevtool/src/task/meta_node_service.rs
index 8dfebbdf4fa3c..b3f55aa371b87 100644
--- a/src/risedevtool/src/task/meta_node_service.rs
+++ b/src/risedevtool/src/task/meta_node_service.rs
@@ -82,7 +82,10 @@ impl MetaNodeService {
 
         match &config.meta_backend {
             MetaBackend::Memory => {
-                cmd.arg("--backend").arg("mem");
+                cmd.arg("--backend")
+                    .arg("sql")
+                    .arg("--sql-endpoint")
+                    .arg("sqlite::memory:");
             }
             MetaBackend::Etcd => {
                 let etcd_config = config.provide_etcd_backend.as_ref().unwrap();
diff --git a/src/risedevtool/src/task/schema_registry_service.rs b/src/risedevtool/src/task/schema_registry_service.rs
index 5c5eba4fa8f35..0f37c732452ce 100644
--- a/src/risedevtool/src/task/schema_registry_service.rs
+++ b/src/risedevtool/src/task/schema_registry_service.rs
@@ -15,6 +15,9 @@
 use super::docker_service::{DockerService, DockerServiceConfig};
 use crate::SchemaRegistryConfig;
 
+/// Schema Registry listener port in the container.
+const SCHEMA_REGISTRY_LISTENER_PORT: &str = "8081";
+
 impl DockerServiceConfig for SchemaRegistryConfig {
     fn id(&self) -> String {
         self.id.clone()
@@ -39,11 +42,14 @@ impl DockerServiceConfig for SchemaRegistryConfig {
             panic!("More than one Kafka is not supported yet");
         }
         let kafka = &kafka[0];
+        if kafka.user_managed {
+            panic!("user-managed Kafka with docker Schema Registry is not supported yet. Please make them both or neither user-managed.");
+        }
         vec![
             ("SCHEMA_REGISTRY_HOST_NAME".to_owned(), self.address.clone()),
             (
                 "SCHEMA_REGISTRY_LISTENERS".to_owned(),
-                format!("http://{}:{}", self.address, self.port),
+                format!("http://{}:{}", "0.0.0.0", SCHEMA_REGISTRY_LISTENER_PORT),
             ),
             (
                 "SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS".to_owned(),
@@ -53,7 +59,10 @@ impl DockerServiceConfig for SchemaRegistryConfig {
     }
 
     fn ports(&self) -> Vec<(String, String)> {
-        vec![(self.port.to_string(), "8081".to_owned())]
+        vec![(
+            self.port.to_string(),
+            SCHEMA_REGISTRY_LISTENER_PORT.to_owned(),
+        )]
     }
 
     fn data_path(&self) -> Option<String> {
diff --git a/src/risedevtool/src/task/sql_server_service.rs b/src/risedevtool/src/task/sql_server_service.rs
new file mode 100644
index 0000000000000..a00e10b7a721a
--- /dev/null
+++ b/src/risedevtool/src/task/sql_server_service.rs
@@ -0,0 +1,50 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::task::docker_service::{DockerService, DockerServiceConfig};
+use crate::SqlServerConfig;
+
+impl DockerServiceConfig for SqlServerConfig {
+    fn id(&self) -> String {
+        self.id.clone()
+    }
+
+    fn is_user_managed(&self) -> bool {
+        self.user_managed
+    }
+
+    fn image(&self) -> String {
+        self.image.clone()
+    }
+
+    fn envs(&self) -> Vec<(String, String)> {
+        vec![
+            ("ACCEPT_EULA".to_owned(), "Y".to_owned()),
+            ("MSSQL_AGENT_ENABLED".to_owned(), "true".to_owned()),
+            ("MSSQL_SA_PASSWORD".to_owned(), self.password.clone()),
+        ]
+    }
+
+    fn ports(&self) -> Vec<(String, String)> {
+        vec![(self.port.to_string(), "1433".to_owned())]
+    }
+
+    fn data_path(&self) -> Option<String> {
+        self.persist_data
+            .then(|| "/var/lib/sqlserver/data".to_owned())
+    }
+}
+
+/// Docker-backed Sql Server service.
+pub type SqlServerService = DockerService<SqlServerConfig>;
diff --git a/src/rpc_client/Cargo.toml b/src/rpc_client/Cargo.toml
index 70bebfa154fc4..37064df273ed0 100644
--- a/src/rpc_client/Cargo.toml
+++ b/src/rpc_client/Cargo.toml
@@ -17,7 +17,7 @@ normal = ["workspace-hack"]
 anyhow = "1"
 async-trait = "0.1"
 easy-ext = "1"
-either = "1.12.0"
+either = "1.13.0"
 futures = { version = "0.3", default-features = false, features = ["alloc"] }
 http = "0.2"
 hyper = "0.14" # required by tonic
diff --git a/src/rpc_client/src/compute_client.rs b/src/rpc_client/src/compute_client.rs
index f908bb21aa3a2..c065bb6935954 100644
--- a/src/rpc_client/src/compute_client.rs
+++ b/src/rpc_client/src/compute_client.rs
@@ -18,7 +18,7 @@ use std::time::Duration;
 use async_trait::async_trait;
 use futures::StreamExt;
 use risingwave_common::config::{MAX_CONNECTION_WINDOW_SIZE, STREAM_WINDOW_SIZE};
-use risingwave_common::monitor::connection::{EndpointExt, TcpConfig};
+use risingwave_common::monitor::{EndpointExt, TcpConfig};
 use risingwave_common::util::addr::HostAddr;
 use risingwave_common::util::tracing::TracingContext;
 use risingwave_pb::batch_plan::{PlanFragment, TaskId, TaskOutputId};
@@ -48,6 +48,12 @@ use tonic::Streaming;
 use crate::error::{Result, RpcError};
 use crate::{RpcClient, RpcClientPool};
 
+// TODO: this client has too many roles, e.g.
+// - batch MPP task query execution
+// - batch exchange
+// - streaming exchange
+// - general services specific to compute node, like monitoring, profiling, debugging, etc.
+// We should consider splitting them into different clients.
 #[derive(Clone)]
 pub struct ComputeClient {
     pub exchange_client: ExchangeServiceClient<Channel>,
@@ -278,4 +284,4 @@ impl RpcClient for ComputeClient {
 }
 
 pub type ComputeClientPool = RpcClientPool<ComputeClient>;
-pub type ComputeClientPoolRef = Arc<ComputeClientPool>;
+pub type ComputeClientPoolRef = Arc<ComputeClientPool>; // TODO: no need for `Arc` since clone is cheap and shared
diff --git a/src/rpc_client/src/connector_client.rs b/src/rpc_client/src/connector_client.rs
index 30d78290b6d98..c81a74d2fa709 100644
--- a/src/rpc_client/src/connector_client.rs
+++ b/src/rpc_client/src/connector_client.rs
@@ -19,7 +19,7 @@ use std::time::Duration;
 use anyhow::{anyhow, Context};
 use futures::TryStreamExt;
 use risingwave_common::config::{MAX_CONNECTION_WINDOW_SIZE, STREAM_WINDOW_SIZE};
-use risingwave_common::monitor::connection::{EndpointExt, TcpConfig};
+use risingwave_common::monitor::{EndpointExt, TcpConfig};
 use risingwave_pb::connector_service::connector_service_client::ConnectorServiceClient;
 use risingwave_pb::connector_service::sink_coordinator_stream_request::{
     CommitMetadata, StartCoordinator,
diff --git a/src/rpc_client/src/error.rs b/src/rpc_client/src/error.rs
index 5626912c2f88b..c5c5613a32a4b 100644
--- a/src/rpc_client/src/error.rs
+++ b/src/rpc_client/src/error.rs
@@ -28,7 +28,14 @@ pub enum RpcError {
     TransportError(Box<tonic::transport::Error>),
 
     #[error(transparent)]
-    GrpcStatus(Box<TonicStatusWrapper>),
+    GrpcStatus(
+        #[from]
+        // Typically it does not have a backtrace,
+        // but this is to let `thiserror` generate `provide` implementation to make `Extra` work.
+        // See `risingwave_error::tonic::extra`.
+        #[backtrace]
+        Box<TonicStatusWrapper>,
+    ),
 
     #[error(transparent)]
     MetaAddressParse(#[from] MetaAddressStrategyParseError),
@@ -61,7 +68,7 @@ macro_rules! impl_from_status {
                 $(
                     #[doc = "Convert a gRPC status from " $service " service into an [`RpcError`]."]
                     pub fn [<from_ $service _status>](s: tonic::Status) -> Self {
-                        Self::grpc_status(s.with_client_side_service_name(stringify!($service)))
+                        Box::new(s.with_client_side_service_name(stringify!($service))).into()
                     }
                 )*
             }
diff --git a/src/rpc_client/src/hummock_meta_client.rs b/src/rpc_client/src/hummock_meta_client.rs
index 6e1dfec3b7be3..5c25a59afa7f8 100644
--- a/src/rpc_client/src/hummock_meta_client.rs
+++ b/src/rpc_client/src/hummock_meta_client.rs
@@ -19,7 +19,8 @@ use risingwave_hummock_sdk::{
     HummockEpoch, HummockSstableObjectId, HummockVersionId, SstObjectIdRange, SyncResult,
 };
 use risingwave_pb::hummock::{
-    HummockSnapshot, SubscribeCompactionEventRequest, SubscribeCompactionEventResponse, VacuumTask,
+    HummockSnapshot, PbHummockVersion, SubscribeCompactionEventRequest,
+    SubscribeCompactionEventResponse, VacuumTask,
 };
 use tokio::sync::mpsc::UnboundedSender;
 
@@ -53,7 +54,11 @@ pub trait HummockMetaClient: Send + Sync + 'static {
         total_object_count: u64,
         total_object_size: u64,
     ) -> Result<()>;
-    async fn trigger_full_gc(&self, sst_retention_time_sec: u64) -> Result<()>;
+    async fn trigger_full_gc(
+        &self,
+        sst_retention_time_sec: u64,
+        prefix: Option<String>,
+    ) -> Result<()>;
 
     async fn subscribe_compaction_event(
         &self,
@@ -61,4 +66,6 @@ pub trait HummockMetaClient: Send + Sync + 'static {
         UnboundedSender<SubscribeCompactionEventRequest>,
         BoxStream<'static, CompactionEventItem>,
     )>;
+
+    async fn get_version_by_epoch(&self, epoch: HummockEpoch) -> Result<PbHummockVersion>;
 }
diff --git a/src/rpc_client/src/lib.rs b/src/rpc_client/src/lib.rs
index fa276bdd0a5ce..bb1d90dcffbf4 100644
--- a/src/rpc_client/src/lib.rs
+++ b/src/rpc_client/src/lib.rs
@@ -88,19 +88,25 @@ pub struct RpcClientPool<S> {
     clients: Cache<HostAddr, Arc<Vec<S>>>,
 }
 
-impl<S> Default for RpcClientPool<S>
-where
-    S: RpcClient,
-{
-    fn default() -> Self {
-        Self::new(1)
+impl<S> std::fmt::Debug for RpcClientPool<S> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("RpcClientPool")
+            .field("connection_pool_size", &self.connection_pool_size)
+            .field("type", &type_name::<S>())
+            .field("len", &self.clients.entry_count())
+            .finish()
     }
 }
 
+/// Intentionally not implementing `Default` to let callers be explicit about the pool size.
+impl<S> !Default for RpcClientPool<S> {}
+
 impl<S> RpcClientPool<S>
 where
     S: RpcClient,
 {
+    /// Create a new pool with the given `connection_pool_size`, which is the number of
+    /// connections to each node that will be reused.
     pub fn new(connection_pool_size: u16) -> Self {
         Self {
             connection_pool_size,
@@ -108,6 +114,16 @@ where
         }
     }
 
+    /// Create a pool for testing purposes. Same as [`Self::adhoc`].
+    pub fn for_test() -> Self {
+        Self::adhoc()
+    }
+
+    /// Create a pool for ad-hoc usage, where the number of connections to each node is 1.
+    pub fn adhoc() -> Self {
+        Self::new(1)
+    }
+
     /// Gets the RPC client for the given node. If the connection is not established, a
     /// new client will be created and returned.
     pub async fn get(&self, node: &WorkerNode) -> Result<S> {
diff --git a/src/rpc_client/src/meta_client.rs b/src/rpc_client/src/meta_client.rs
index df17a6e7cb9ec..951ed56f64fed 100644
--- a/src/rpc_client/src/meta_client.rs
+++ b/src/rpc_client/src/meta_client.rs
@@ -14,6 +14,8 @@
 
 use std::collections::HashMap;
 use std::fmt::{Debug, Display};
+use std::sync::atomic::AtomicBool;
+use std::sync::atomic::Ordering::Relaxed;
 use std::sync::Arc;
 use std::thread;
 use std::time::{Duration, SystemTime};
@@ -51,6 +53,7 @@ use risingwave_pb::cloud_service::*;
 use risingwave_pb::common::{HostAddress, WorkerNode, WorkerType};
 use risingwave_pb::connector_service::sink_coordination_service_client::SinkCoordinationServiceClient;
 use risingwave_pb::ddl_service::alter_owner_request::Object;
+use risingwave_pb::ddl_service::create_materialized_view_request::PbBackfillType;
 use risingwave_pb::ddl_service::ddl_service_client::DdlServiceClient;
 use risingwave_pb::ddl_service::drop_table_request::SourceId;
 use risingwave_pb::ddl_service::*;
@@ -65,7 +68,6 @@ use risingwave_pb::meta::add_worker_node_request::Property;
 use risingwave_pb::meta::cancel_creating_jobs_request::PbJobs;
 use risingwave_pb::meta::cluster_service_client::ClusterServiceClient;
 use risingwave_pb::meta::event_log_service_client::EventLogServiceClient;
-use risingwave_pb::meta::get_reschedule_plan_request::PbPolicy;
 use risingwave_pb::meta::heartbeat_request::{extra_info, ExtraInfo};
 use risingwave_pb::meta::heartbeat_service_client::HeartbeatServiceClient;
 use risingwave_pb::meta::list_actor_states_response::ActorState;
@@ -115,6 +117,8 @@ pub struct MetaClient {
     host_addr: HostAddr,
     inner: GrpcMetaClient,
     meta_config: MetaConfig,
+    cluster_id: String,
+    shutting_down: Arc<AtomicBool>,
 }
 
 impl MetaClient {
@@ -130,6 +134,10 @@ impl MetaClient {
         self.worker_type
     }
 
+    pub fn cluster_id(&self) -> &str {
+        &self.cluster_id
+    }
+
     /// Subscribe to notification from meta.
     pub async fn subscribe(
         &self,
@@ -271,6 +279,8 @@ impl MetaClient {
             host_addr: addr.clone(),
             inner: grpc_meta_client,
             meta_config: meta_config.to_owned(),
+            cluster_id: add_worker_resp.cluster_id,
+            shutting_down: Arc::new(false.into()),
         };
 
         static REPORT_PANIC: std::sync::Once = std::sync::Once::new();
@@ -317,8 +327,12 @@ impl MetaClient {
         let resp = self.inner.heartbeat(request).await?;
         if let Some(status) = resp.status {
             if status.code() == risingwave_pb::common::status::Code::UnknownWorker {
-                tracing::error!("worker expired: {}", status.message);
-                std::process::exit(1);
+                // Ignore the error if we're already shutting down.
+                // Otherwise, exit the process.
+                if !self.shutting_down.load(Relaxed) {
+                    tracing::error!(message = status.message, "worker expired");
+                    std::process::exit(1);
+                }
             }
         }
         Ok(())
@@ -348,6 +362,7 @@ impl MetaClient {
         let request = CreateMaterializedViewRequest {
             materialized_view: Some(table),
             fragment_graph: Some(graph),
+            backfill: PbBackfillType::Regular as _,
         };
         let resp = self.inner.create_materialized_view(request).await?;
         // TODO: handle error in `resp.status` here
@@ -528,6 +543,7 @@ impl MetaClient {
         table: PbTable,
         graph: StreamFragmentGraph,
         table_col_index_mapping: ColIndexMapping,
+        job_type: PbTableJobType,
     ) -> Result<CatalogVersion> {
         let request = ReplaceTablePlanRequest {
             plan: Some(ReplaceTablePlan {
@@ -535,6 +551,7 @@ impl MetaClient {
                 table: Some(table),
                 fragment_graph: Some(graph),
                 table_col_index_mapping: Some(table_col_index_mapping.to_protobuf()),
+                job_type: job_type as _,
             }),
         };
         let resp = self.inner.replace_table_plan(request).await?;
@@ -738,6 +755,7 @@ impl MetaClient {
             host: Some(self.host_addr.to_protobuf()),
         };
         self.inner.delete_worker_node(request).await?;
+        self.shutting_down.store(true, Relaxed);
         Ok(())
     }
 
@@ -939,6 +957,14 @@ impl MetaClient {
         Ok(resp)
     }
 
+    pub async fn get_cluster_recovery_status(&self) -> Result<RecoveryStatus> {
+        let resp = self
+            .inner
+            .get_cluster_recovery_status(GetClusterRecoveryStatusRequest {})
+            .await?;
+        Ok(resp.get_status().unwrap())
+    }
+
     pub async fn get_cluster_info(&self) -> Result<GetClusterInfoResponse> {
         let request = GetClusterInfoRequest {};
         let resp = self.inner.get_cluster_info(request).await?;
@@ -947,32 +973,19 @@ impl MetaClient {
 
     pub async fn reschedule(
         &self,
-        reschedules: HashMap<u32, PbReschedule>,
+        worker_reschedules: HashMap<u32, PbWorkerReschedule>,
         revision: u64,
         resolve_no_shuffle_upstream: bool,
     ) -> Result<(bool, u64)> {
         let request = RescheduleRequest {
-            reschedules,
             revision,
             resolve_no_shuffle_upstream,
+            worker_reschedules,
         };
         let resp = self.inner.reschedule(request).await?;
         Ok((resp.success, resp.revision))
     }
 
-    pub async fn get_reschedule_plan(
-        &self,
-        policy: PbPolicy,
-        revision: u64,
-    ) -> Result<GetReschedulePlanResponse> {
-        let request = GetReschedulePlanRequest {
-            revision,
-            policy: Some(policy),
-        };
-        let resp = self.inner.get_reschedule_plan(request).await?;
-        Ok(resp)
-    }
-
     pub async fn risectl_get_pinned_versions_summary(
         &self,
     ) -> Result<RiseCtlGetPinnedVersionsSummaryResponse> {
@@ -1030,7 +1043,7 @@ impl MetaClient {
         version_delta: HummockVersionDelta,
     ) -> Result<(HummockVersion, Vec<CompactionGroupId>)> {
         let req = ReplayVersionDeltaRequest {
-            version_delta: Some(version_delta.to_protobuf()),
+            version_delta: Some(version_delta.into()),
         };
         let resp = self.inner.replay_version_delta(req).await?;
         Ok((
@@ -1383,6 +1396,12 @@ impl MetaClient {
         let resp = self.inner.cancel_compact_task(req).await?;
         Ok(resp.ret)
     }
+
+    pub async fn get_version_by_epoch(&self, epoch: HummockEpoch) -> Result<PbHummockVersion> {
+        let req = GetVersionByEpochRequest { epoch };
+        let resp = self.inner.get_version_by_epoch(req).await?;
+        Ok(resp.version.unwrap())
+    }
 }
 
 #[async_trait]
@@ -1504,10 +1523,15 @@ impl HummockMetaClient for MetaClient {
         Ok(())
     }
 
-    async fn trigger_full_gc(&self, sst_retention_time_sec: u64) -> Result<()> {
+    async fn trigger_full_gc(
+        &self,
+        sst_retention_time_sec: u64,
+        prefix: Option<String>,
+    ) -> Result<()> {
         self.inner
             .trigger_full_gc(TriggerFullGcRequest {
                 sst_retention_time_sec,
+                prefix,
             })
             .await?;
         Ok(())
@@ -1544,6 +1568,10 @@ impl HummockMetaClient for MetaClient {
 
         Ok((request_sender, Box::pin(stream)))
     }
+
+    async fn get_version_by_epoch(&self, epoch: HummockEpoch) -> Result<PbHummockVersion> {
+        self.get_version_by_epoch(epoch).await
+    }
 }
 
 #[async_trait]
@@ -1764,9 +1792,9 @@ impl GrpcMetaClient {
     // See `Endpoint::keep_alive_timeout`
     const ENDPOINT_KEEP_ALIVE_TIMEOUT_SEC: u64 = 60;
     // Retry base interval in ms for connecting to meta server.
-    const INIT_RETRY_BASE_INTERVAL_MS: u64 = 50;
+    const INIT_RETRY_BASE_INTERVAL_MS: u64 = 10;
     // Max retry times for connecting to meta server.
-    const INIT_RETRY_MAX_INTERVAL_MS: u64 = 5000;
+    const INIT_RETRY_MAX_INTERVAL_MS: u64 = 2000;
 
     fn start_meta_member_monitor(
         &self,
@@ -1953,6 +1981,7 @@ macro_rules! for_all_meta_rpc {
             ,{ cluster_client, delete_worker_node, DeleteWorkerNodeRequest, DeleteWorkerNodeResponse }
             ,{ cluster_client, update_worker_node_schedulability, UpdateWorkerNodeSchedulabilityRequest, UpdateWorkerNodeSchedulabilityResponse }
             ,{ cluster_client, list_all_nodes, ListAllNodesRequest, ListAllNodesResponse }
+            ,{ cluster_client, get_cluster_recovery_status, GetClusterRecoveryStatusRequest, GetClusterRecoveryStatusResponse }
             ,{ heartbeat_client, heartbeat, HeartbeatRequest, HeartbeatResponse }
             ,{ stream_client, flush, FlushRequest, FlushResponse }
             ,{ stream_client, pause, PauseRequest, PauseResponse }
@@ -2038,6 +2067,7 @@ macro_rules! for_all_meta_rpc {
             ,{ hummock_client, list_compact_task_progress, ListCompactTaskProgressRequest, ListCompactTaskProgressResponse }
             ,{ hummock_client, cancel_compact_task, CancelCompactTaskRequest, CancelCompactTaskResponse}
             ,{ hummock_client, list_change_log_epochs, ListChangeLogEpochsRequest, ListChangeLogEpochsResponse }
+            ,{ hummock_client, get_version_by_epoch, GetVersionByEpochRequest, GetVersionByEpochResponse }
             ,{ user_client, create_user, CreateUserRequest, CreateUserResponse }
             ,{ user_client, update_user, UpdateUserRequest, UpdateUserResponse }
             ,{ user_client, drop_user, DropUserRequest, DropUserResponse }
@@ -2045,7 +2075,6 @@ macro_rules! for_all_meta_rpc {
             ,{ user_client, revoke_privilege, RevokePrivilegeRequest, RevokePrivilegeResponse }
             ,{ scale_client, get_cluster_info, GetClusterInfoRequest, GetClusterInfoResponse }
             ,{ scale_client, reschedule, RescheduleRequest, RescheduleResponse }
-            ,{ scale_client, get_reschedule_plan, GetReschedulePlanRequest, GetReschedulePlanResponse }
             ,{ notification_client, subscribe, SubscribeRequest, Streaming<SubscribeResponse> }
             ,{ backup_client, backup_meta, BackupMetaRequest, BackupMetaResponse }
             ,{ backup_client, get_backup_job_status, GetBackupJobStatusRequest, GetBackupJobStatusResponse }
diff --git a/src/rpc_client/src/sink_coordinate_client.rs b/src/rpc_client/src/sink_coordinate_client.rs
index 74c05fa85de8e..06602ef4db3b7 100644
--- a/src/rpc_client/src/sink_coordinate_client.rs
+++ b/src/rpc_client/src/sink_coordinate_client.rs
@@ -16,7 +16,7 @@ use std::future::Future;
 
 use anyhow::anyhow;
 use futures::{Stream, TryStreamExt};
-use risingwave_common::buffer::Bitmap;
+use risingwave_common::bitmap::Bitmap;
 use risingwave_pb::connector_service::coordinate_request::{
     CommitRequest, StartCoordinationRequest,
 };
diff --git a/src/rpc_client/src/stream_client.rs b/src/rpc_client/src/stream_client.rs
index 4710be7085ef6..988931cb207b6 100644
--- a/src/rpc_client/src/stream_client.rs
+++ b/src/rpc_client/src/stream_client.rs
@@ -19,7 +19,7 @@ use anyhow::anyhow;
 use async_trait::async_trait;
 use futures::TryStreamExt;
 use risingwave_common::config::MAX_CONNECTION_WINDOW_SIZE;
-use risingwave_common::monitor::connection::{EndpointExt, TcpConfig};
+use risingwave_common::monitor::{EndpointExt, TcpConfig};
 use risingwave_common::util::addr::HostAddr;
 use risingwave_pb::stream_service::stream_service_client::StreamServiceClient;
 use risingwave_pb::stream_service::streaming_control_stream_request::InitRequest;
diff --git a/src/sqlparser/src/ast/ddl.rs b/src/sqlparser/src/ast/ddl.rs
index 92683f42e742a..6ea385df950fc 100644
--- a/src/sqlparser/src/ast/ddl.rs
+++ b/src/sqlparser/src/ast/ddl.rs
@@ -102,8 +102,8 @@ pub enum AlterTableOperation {
         deferred: bool,
     },
     RefreshSchema,
-    /// `SET STREAMING_RATE_LIMIT TO <rate_limit>`
-    SetStreamingRateLimit {
+    /// `SET SOURCE_RATE_LIMIT TO <rate_limit>`
+    SetSourceRateLimit {
         rate_limit: i32,
     },
 }
@@ -138,8 +138,8 @@ pub enum AlterViewOperation {
         parallelism: SetVariableValue,
         deferred: bool,
     },
-    /// `SET STREAMING_RATE_LIMIT TO <rate_limit>`
-    SetStreamingRateLimit {
+    /// `SET BACKFILL_RATE_LIMIT TO <rate_limit>`
+    SetBackfillRateLimit {
         rate_limit: i32,
     },
 }
@@ -180,7 +180,7 @@ pub enum AlterSourceOperation {
     SetSchema { new_schema_name: ObjectName },
     FormatEncode { connector_schema: ConnectorSchema },
     RefreshSchema,
-    SetStreamingRateLimit { rate_limit: i32 },
+    SetSourceRateLimit { rate_limit: i32 },
 }
 
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
@@ -290,8 +290,8 @@ impl fmt::Display for AlterTableOperation {
             AlterTableOperation::RefreshSchema => {
                 write!(f, "REFRESH SCHEMA")
             }
-            AlterTableOperation::SetStreamingRateLimit { rate_limit } => {
-                write!(f, "SET STREAMING_RATE_LIMIT TO {}", rate_limit)
+            AlterTableOperation::SetSourceRateLimit { rate_limit } => {
+                write!(f, "SET SOURCE_RATE_LIMIT TO {}", rate_limit)
             }
         }
     }
@@ -341,8 +341,8 @@ impl fmt::Display for AlterViewOperation {
                     if *deferred { " DEFERRED" } else { "" }
                 )
             }
-            AlterViewOperation::SetStreamingRateLimit { rate_limit } => {
-                write!(f, "SET STREAMING_RATE_LIMIT TO {}", rate_limit)
+            AlterViewOperation::SetBackfillRateLimit { rate_limit } => {
+                write!(f, "SET BACKFILL_RATE_LIMIT TO {}", rate_limit)
             }
         }
     }
@@ -412,8 +412,8 @@ impl fmt::Display for AlterSourceOperation {
             AlterSourceOperation::RefreshSchema => {
                 write!(f, "REFRESH SCHEMA")
             }
-            AlterSourceOperation::SetStreamingRateLimit { rate_limit } => {
-                write!(f, "SET STREAMING_RATE_LIMIT TO {}", rate_limit)
+            AlterSourceOperation::SetSourceRateLimit { rate_limit } => {
+                write!(f, "SET SOURCE_RATE_LIMIT TO {}", rate_limit)
             }
         }
     }
diff --git a/src/sqlparser/src/ast/mod.rs b/src/sqlparser/src/ast/mod.rs
index db86c6fb8a47e..d5cca61b6a186 100644
--- a/src/sqlparser/src/ast/mod.rs
+++ b/src/sqlparser/src/ast/mod.rs
@@ -46,14 +46,14 @@ pub use self::legacy_source::{
 };
 pub use self::operator::{BinaryOperator, QualifiedOperator, UnaryOperator};
 pub use self::query::{
-    Cte, CteInner, Distinct, Fetch, Join, JoinConstraint, JoinOperator, LateralView, OrderByExpr,
-    Query, Select, SelectItem, SetExpr, SetOperator, TableAlias, TableFactor, TableWithJoins, Top,
-    Values, With,
+    Corresponding, Cte, CteInner, Distinct, Fetch, Join, JoinConstraint, JoinOperator, LateralView,
+    OrderByExpr, Query, Select, SelectItem, SetExpr, SetOperator, TableAlias, TableFactor,
+    TableWithJoins, Top, Values, With,
 };
 pub use self::statement::*;
 pub use self::value::{
-    CstyleEscapedString, DateTimeField, DollarQuotedString, JsonPredicateType, TrimWhereField,
-    Value,
+    CstyleEscapedString, DateTimeField, DollarQuotedString, JsonPredicateType, SecretRef,
+    SecretRefAsType, TrimWhereField, Value,
 };
 pub use crate::ast::ddl::{
     AlterIndexOperation, AlterSinkOperation, AlterSourceOperation, AlterSubscriptionOperation,
@@ -2481,6 +2481,8 @@ impl fmt::Display for FunctionArg {
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
 #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
 pub struct Function {
+    /// Whether the function is prefixed with `aggregate:`
+    pub scalar_as_agg: bool,
     pub name: ObjectName,
     pub args: Vec<FunctionArg>,
     /// whether the last argument is variadic, e.g. `foo(a, b, variadic c)`
@@ -2497,6 +2499,7 @@ pub struct Function {
 impl Function {
     pub fn no_arg(name: ObjectName) -> Self {
         Self {
+            scalar_as_agg: false,
             name,
             args: vec![],
             variadic: false,
@@ -2511,6 +2514,9 @@ impl Function {
 
 impl fmt::Display for Function {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        if self.scalar_as_agg {
+            write!(f, "aggregate:")?;
+        }
         write!(
             f,
             "{}({}",
@@ -3153,6 +3159,8 @@ impl fmt::Display for SetVariableValueSingle {
 #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
 pub enum AsOf {
     ProcessTime,
+    // used by time travel
+    ProcessTimeWithInterval((String, DateTimeField)),
     // the number of seconds that have elapsed since the Unix epoch, which is January 1, 1970 at 00:00:00 Coordinated Universal Time (UTC).
     TimestampNum(i64),
     TimestampString(String),
@@ -3165,6 +3173,11 @@ impl fmt::Display for AsOf {
         use AsOf::*;
         match self {
             ProcessTime => write!(f, " FOR SYSTEM_TIME AS OF PROCTIME()"),
+            ProcessTimeWithInterval((value, leading_field)) => write!(
+                f,
+                " FOR SYSTEM_TIME AS OF NOW() - {} {}",
+                value, leading_field
+            ),
             TimestampNum(ts) => write!(f, " FOR SYSTEM_TIME AS OF {}", ts),
             TimestampString(ts) => write!(f, " FOR SYSTEM_TIME AS OF '{}'", ts),
             VersionNum(v) => write!(f, " FOR SYSTEM_VERSION AS OF {}", v),
diff --git a/src/sqlparser/src/ast/query.rs b/src/sqlparser/src/ast/query.rs
index 83e84907a1091..b16a3075f90d9 100644
--- a/src/sqlparser/src/ast/query.rs
+++ b/src/sqlparser/src/ast/query.rs
@@ -97,6 +97,7 @@ pub enum SetExpr {
     SetOperation {
         op: SetOperator,
         all: bool,
+        corresponding: Corresponding,
         left: Box<SetExpr>,
         right: Box<SetExpr>,
     },
@@ -114,9 +115,10 @@ impl fmt::Display for SetExpr {
                 right,
                 op,
                 all,
+                corresponding,
             } => {
                 let all_str = if *all { " ALL" } else { "" };
-                write!(f, "{} {}{} {}", left, op, all_str, right)
+                write!(f, "{} {}{}{} {}", left, op, all_str, corresponding, right)
             }
         }
     }
@@ -140,6 +142,50 @@ impl fmt::Display for SetOperator {
     }
 }
 
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
+/// `CORRESPONDING [ BY <left paren> <corresponding column list> <right paren> ]`
+pub struct Corresponding {
+    pub corresponding: bool,
+    pub column_list: Option<Vec<Ident>>,
+}
+
+impl Corresponding {
+    pub fn with_column_list(column_list: Option<Vec<Ident>>) -> Self {
+        Self {
+            corresponding: true,
+            column_list,
+        }
+    }
+
+    pub fn none() -> Self {
+        Self {
+            corresponding: false,
+            column_list: None,
+        }
+    }
+
+    pub fn is_corresponding(&self) -> bool {
+        self.corresponding
+    }
+
+    pub fn column_list(&self) -> Option<&[Ident]> {
+        self.column_list.as_deref()
+    }
+}
+
+impl fmt::Display for Corresponding {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        if self.corresponding {
+            write!(f, " CORRESPONDING")?;
+            if let Some(column_list) = &self.column_list {
+                write!(f, " BY ({})", display_comma_separated(column_list))?;
+            }
+        }
+        Ok(())
+    }
+}
+
 /// A restricted variant of `SELECT` (without CTEs/`ORDER BY`), which may
 /// appear either as the only body item of an `SQLQuery`, or as an operand
 /// to a set operation like `UNION`.
diff --git a/src/sqlparser/src/ast/statement.rs b/src/sqlparser/src/ast/statement.rs
index facedadeb5bc0..732badfdd0a52 100644
--- a/src/sqlparser/src/ast/statement.rs
+++ b/src/sqlparser/src/ast/statement.rs
@@ -173,6 +173,7 @@ pub enum Encode {
     /// Used internally for schema change
     Native,
     Template,
+    Parquet,
 }
 
 // TODO: unify with `from_keyword`
@@ -190,6 +191,7 @@ impl fmt::Display for Encode {
                 Encode::Native => "NATIVE",
                 Encode::Template => "TEMPLATE",
                 Encode::None => "NONE",
+                Encode::Parquet => "PARQUET",
                 Encode::Text => "TEXT",
             }
         )
@@ -208,8 +210,9 @@ impl Encode {
             "TEMPLATE" => Encode::Template,
             "NATIVE" => Encode::Native,
             "NONE" => Encode::None,
+            "PARQUET" => Encode::Parquet,
             _ => parser_err!(
-                "expected AVRO | BYTES | CSV | PROTOBUF | JSON | NATIVE | TEMPLATE | NONE after Encode"
+                "expected AVRO | BYTES | CSV | PROTOBUF | JSON | NATIVE | TEMPLATE | PARQUET | NONE after Encode"
             ),
         })
     }
diff --git a/src/sqlparser/src/ast/value.rs b/src/sqlparser/src/ast/value.rs
index 79f2a6ebd99ca..9cae715e09278 100644
--- a/src/sqlparser/src/ast/value.rs
+++ b/src/sqlparser/src/ast/value.rs
@@ -60,7 +60,7 @@ pub enum Value {
     /// `NULL` value
     Null,
     /// name of the reference to secret
-    Ref(ObjectName),
+    Ref(SecretRef),
 }
 
 impl fmt::Display for Value {
@@ -115,7 +115,7 @@ impl fmt::Display for Value {
                 Ok(())
             }
             Value::Null => write!(f, "NULL"),
-            Value::Ref(v) => write!(f, "ref secret {}", v),
+            Value::Ref(v) => write!(f, "secret {}", v),
         }
     }
 }
@@ -238,3 +238,25 @@ impl fmt::Display for JsonPredicateType {
         })
     }
 }
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
+pub struct SecretRef {
+    pub secret_name: ObjectName,
+    pub ref_as: SecretRefAsType,
+}
+
+impl fmt::Display for SecretRef {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self.ref_as {
+            SecretRefAsType::Text => write!(f, "{}", self.secret_name),
+            SecretRefAsType::File => write!(f, "{} AS FILE", self.secret_name),
+        }
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
+pub enum SecretRefAsType {
+    Text,
+    File,
+}
diff --git a/src/sqlparser/src/keywords.rs b/src/sqlparser/src/keywords.rs
index fc5971bf4640b..768301544ef21 100644
--- a/src/sqlparser/src/keywords.rs
+++ b/src/sqlparser/src/keywords.rs
@@ -230,6 +230,7 @@ define_keywords!(
     EXTRACT,
     FALSE,
     FETCH,
+    FILE,
     FILTER,
     FIRST,
     FIRST_VALUE,
@@ -488,7 +489,6 @@ define_keywords!(
     STDDEV_SAMP,
     STDIN,
     STORED,
-    STREAMING_RATE_LIMIT,
     STRING,
     STRUCT,
     SUBMULTISET,
diff --git a/src/sqlparser/src/parser.rs b/src/sqlparser/src/parser.rs
index 60e9da014f10d..996fd9ebe8490 100644
--- a/src/sqlparser/src/parser.rs
+++ b/src/sqlparser/src/parser.rs
@@ -31,7 +31,8 @@ use crate::ast::*;
 use crate::keywords::{self, Keyword};
 use crate::parser_v2;
 use crate::parser_v2::{
-    dollar_quoted_string, keyword, literal_i64, literal_uint, single_quoted_string, ParserExt as _,
+    dollar_quoted_string, keyword, literal_i64, literal_uint, single_quoted_string, token_number,
+    ParserExt as _,
 };
 use crate::tokenizer::*;
 
@@ -609,7 +610,8 @@ impl Parser<'_> {
                 Keyword::ARRAY if self.peek_token() == Token::LBracket => self.parse_array_expr(),
                 // `LEFT` and `RIGHT` are reserved as identifier but okay as function
                 Keyword::LEFT | Keyword::RIGHT => {
-                    self.parse_function(ObjectName(vec![w.to_ident()?]))
+                    *self = checkpoint;
+                    self.parse_function()
                 }
                 Keyword::OPERATOR if self.peek_token().token == Token::LParen => {
                     let op = UnaryOperator::PGQualified(Box::new(self.parse_qualified_operator()?));
@@ -643,24 +645,15 @@ impl Parser<'_> {
                 // Here `w` is a word, check if it's a part of a multi-part
                 // identifier, a function call, or a simple identifier:
                 _ => match self.peek_token().token {
-                    Token::LParen | Token::Period => {
-                        let mut id_parts: Vec<Ident> = vec![w.to_ident()?];
-                        while self.consume_token(&Token::Period) {
-                            let ckpt = *self;
-                            let token = self.next_token();
-                            match token.token {
-                                Token::Word(w) => id_parts.push(w.to_ident()?),
-                                _ => {
-                                    *self = ckpt;
-                                    return self.expected("an identifier or a '*' after '.'");
-                                }
-                            }
-                        }
-
-                        if self.peek_token().token == Token::LParen {
-                            self.parse_function(ObjectName(id_parts))
+                    Token::LParen | Token::Period | Token::Colon => {
+                        *self = checkpoint;
+                        if let Ok(object_name) = self.parse_object_name()
+                            && !matches!(self.peek_token().token, Token::LParen | Token::Colon)
+                        {
+                            Ok(Expr::CompoundIdentifier(object_name.0))
                         } else {
-                            Ok(Expr::CompoundIdentifier(id_parts))
+                            *self = checkpoint;
+                            self.parse_function()
                         }
                     }
                     _ => Ok(Expr::Identifier(w.to_ident()?)),
@@ -819,7 +812,16 @@ impl Parser<'_> {
         Ok(QualifiedOperator { schema, name })
     }
 
-    pub fn parse_function(&mut self, name: ObjectName) -> PResult<Expr> {
+    /// Parse a function call.
+    pub fn parse_function(&mut self) -> PResult<Expr> {
+        // [aggregate:]
+        let scalar_as_agg = if self.parse_keyword(Keyword::AGGREGATE) {
+            self.expect_token(&Token::Colon)?;
+            true
+        } else {
+            false
+        };
+        let name = self.parse_object_name()?;
         self.expect_token(&Token::LParen)?;
         let distinct = self.parse_all_or_distinct()?;
         let (args, order_by, variadic) = self.parse_optional_args()?;
@@ -875,6 +877,7 @@ impl Parser<'_> {
         };
 
         Ok(Expr::Function(Function {
+            scalar_as_agg,
             name,
             args,
             variadic,
@@ -1809,6 +1812,18 @@ impl Parser<'_> {
         self.expected(expected)
     }
 
+    /// Check if the expected match is the next token.
+    /// The equality check is case-insensitive.
+    pub fn parse_word(&mut self, expected: &str) -> bool {
+        match self.peek_token().token {
+            Token::Word(w) if w.value.to_uppercase() == expected => {
+                self.next_token();
+                true
+            }
+            _ => false,
+        }
+    }
+
     /// Look for an expected keyword and consume it if it exists
     #[must_use]
     pub fn parse_keyword(&mut self, expected: Keyword) -> bool {
@@ -3075,10 +3090,10 @@ impl Parser<'_> {
                     parallelism: value,
                     deferred,
                 }
-            } else if let Some(rate_limit) = self.parse_alter_streaming_rate_limit()? {
-                AlterTableOperation::SetStreamingRateLimit { rate_limit }
+            } else if let Some(rate_limit) = self.parse_alter_source_rate_limit(true)? {
+                AlterTableOperation::SetSourceRateLimit { rate_limit }
             } else {
-                return self.expected("SCHEMA/PARALLELISM/STREAMING_RATE_LIMIT after SET");
+                return self.expected("SCHEMA/PARALLELISM/SOURCE_RATE_LIMIT after SET");
             }
         } else if self.parse_keyword(Keyword::DROP) {
             let _ = self.parse_keyword(Keyword::COLUMN);
@@ -3130,14 +3145,37 @@ impl Parser<'_> {
         })
     }
 
-    /// STREAMING_RATE_LIMIT = default | NUMBER
-    /// STREAMING_RATE_LIMIT TO default | NUMBER
-    pub fn parse_alter_streaming_rate_limit(&mut self) -> PResult<Option<i32>> {
-        if !self.parse_keyword(Keyword::STREAMING_RATE_LIMIT) {
+    /// BACKFILL_RATE_LIMIT = default | NUMBER
+    /// BACKFILL_RATE_LIMIT TO default | NUMBER
+    pub fn parse_alter_backfill_rate_limit(&mut self) -> PResult<Option<i32>> {
+        if !self.parse_word("BACKFILL_RATE_LIMIT") {
+            return Ok(None);
+        }
+        if self.expect_keyword(Keyword::TO).is_err() && self.expect_token(&Token::Eq).is_err() {
+            return self.expected("TO or = after ALTER TABLE SET BACKFILL_RATE_LIMIT");
+        }
+        let rate_limit = if self.parse_keyword(Keyword::DEFAULT) {
+            -1
+        } else {
+            let s = self.parse_number_value()?;
+            if let Ok(n) = s.parse::<i32>() {
+                n
+            } else {
+                return self.expected("number or DEFAULT");
+            }
+        };
+        Ok(Some(rate_limit))
+    }
+
+    /// SOURCE_RATE_LIMIT = default | NUMBER
+    /// SOURCE_RATE_LIMIT TO default | NUMBER
+    pub fn parse_alter_source_rate_limit(&mut self, is_table: bool) -> PResult<Option<i32>> {
+        if !self.parse_word("SOURCE_RATE_LIMIT") {
             return Ok(None);
         }
         if self.expect_keyword(Keyword::TO).is_err() && self.expect_token(&Token::Eq).is_err() {
-            return self.expected("TO or = after ALTER TABLE SET STREAMING_RATE_LIMIT");
+            let ddl = if is_table { "TABLE" } else { "SOURCE" };
+            return self.expected(&format!("TO or = after ALTER {ddl} SET SOURCE_RATE_LIMIT"));
         }
         let rate_limit = if self.parse_keyword(Keyword::DEFAULT) {
             -1
@@ -3226,11 +3264,11 @@ impl Parser<'_> {
                     deferred,
                 }
             } else if materialized
-                && let Some(rate_limit) = self.parse_alter_streaming_rate_limit()?
+                && let Some(rate_limit) = self.parse_alter_backfill_rate_limit()?
             {
-                AlterViewOperation::SetStreamingRateLimit { rate_limit }
+                AlterViewOperation::SetBackfillRateLimit { rate_limit }
             } else {
-                return self.expected("SCHEMA/PARALLELISM/STREAMING_RATE_LIMIT after SET");
+                return self.expected("SCHEMA/PARALLELISM/BACKFILL_RATE_LIMIT after SET");
             }
         } else {
             return self.expected(&format!(
@@ -3351,8 +3389,8 @@ impl Parser<'_> {
                 AlterSourceOperation::SetSchema {
                     new_schema_name: schema_name,
                 }
-            } else if let Some(rate_limit) = self.parse_alter_streaming_rate_limit()? {
-                AlterSourceOperation::SetStreamingRateLimit { rate_limit }
+            } else if let Some(rate_limit) = self.parse_alter_source_rate_limit(false)? {
+                AlterSourceOperation::SetSourceRateLimit { rate_limit }
             } else {
                 return self.expected("SCHEMA after SET");
             }
@@ -3366,7 +3404,7 @@ impl Parser<'_> {
             AlterSourceOperation::RefreshSchema
         } else {
             return self.expected(
-                "RENAME, ADD COLUMN, OWNER TO, SET or STREAMING_RATE_LIMIT after ALTER SOURCE",
+                "RENAME, ADD COLUMN, OWNER TO, SET or SOURCE_RATE_LIMIT after ALTER SOURCE",
             );
         };
 
@@ -3495,6 +3533,18 @@ impl Parser<'_> {
                     Some('\'') => Ok(Value::SingleQuotedString(w.value)),
                     _ => self.expected_at(checkpoint, "A value")?,
                 },
+                Keyword::SECRET => {
+                    let secret_name = self.parse_object_name()?;
+                    let ref_as = if self.parse_keywords(&[Keyword::AS, Keyword::FILE]) {
+                        SecretRefAsType::File
+                    } else {
+                        SecretRefAsType::Text
+                    };
+                    Ok(Value::Ref(SecretRef {
+                        secret_name,
+                        ref_as,
+                    }))
+                }
                 _ => self.expected_at(checkpoint, "a concrete value"),
             },
             Token::Number(ref n) => Ok(Value::Number(n.clone())),
@@ -3573,23 +3623,13 @@ impl Parser<'_> {
 
     /// Parse a map key string
     pub fn parse_map_key(&mut self) -> PResult<Expr> {
-        let checkpoint = *self;
-        let token = self.next_token();
-        match token.token {
-            Token::Word(Word {
-                value,
-                keyword: Keyword::NoKeyword,
-                ..
-            }) => {
-                if self.peek_token() == Token::LParen {
-                    return self.parse_function(ObjectName(vec![Ident::new_unchecked(value)]));
-                }
-                Ok(Expr::Value(Value::SingleQuotedString(value)))
-            }
-            Token::SingleQuotedString(s) => Ok(Expr::Value(Value::SingleQuotedString(s))),
-            Token::Number(s) => Ok(Expr::Value(Value::Number(s))),
-            _ => self.expected_at(checkpoint, "literal string, number or function"),
-        }
+        alt((
+            Self::parse_function,
+            single_quoted_string.map(|s| Expr::Value(Value::SingleQuotedString(s))),
+            token_number.map(|s| Expr::Value(Value::Number(s))),
+            fail.expect("literal string, number or function"),
+        ))
+        .parse_next(self)
     }
 
     /// Parse a SQL datatype (in the context of a CREATE TABLE statement for example) and convert
@@ -3649,10 +3689,41 @@ impl Parser<'_> {
                 (Keyword::SYSTEM_TIME, Keyword::AS, Keyword::OF),
                 cut_err(
                     alt((
-                        (
-                            Self::parse_identifier.verify(|ident| {
-                                ident.real_value() == "proctime" || ident.real_value() == "now"
+                        preceded(
+                            (
+                                Self::parse_identifier.verify(|ident| ident.real_value() == "now"),
+                                cut_err(Token::LParen),
+                                cut_err(Token::RParen),
+                                Token::Minus,
+                            ),
+                            Self::parse_literal_interval.try_map(|e| match e {
+                                Expr::Value(v) => match v {
+                                    Value::Interval {
+                                        value,
+                                        leading_field,
+                                        ..
+                                    } => {
+                                        let Some(leading_field) = leading_field else {
+                                            return Err(StrError("expect duration unit".into()));
+                                        };
+                                        Ok(AsOf::ProcessTimeWithInterval((value, leading_field)))
+                                    }
+                                    _ => Err(StrError("expect Value::Interval".into())),
+                                },
+                                _ => Err(StrError("expect Expr::Value".into())),
                             }),
+                        ),
+                        (
+                            Self::parse_identifier.verify(|ident| ident.real_value() == "now"),
+                            cut_err(Token::LParen),
+                            cut_err(Token::RParen),
+                        )
+                            .value(AsOf::ProcessTimeWithInterval((
+                                "0".to_owned(),
+                                DateTimeField::Second,
+                            ))),
+                        (
+                            Self::parse_identifier.verify(|ident| ident.real_value() == "proctime"),
                             cut_err(Token::LParen),
                             cut_err(Token::RParen),
                         )
@@ -4043,10 +4114,15 @@ impl Parser<'_> {
                 break;
             }
             self.next_token(); // skip past the set operator
+
+            let all = self.parse_keyword(Keyword::ALL);
+            let corresponding = self.parse_corresponding()?;
+
             expr = SetExpr::SetOperation {
                 left: Box::new(expr),
                 op: op.unwrap(),
-                all: self.parse_keyword(Keyword::ALL),
+                corresponding,
+                all,
                 right: Box::new(self.parse_query_body(next_precedence)?),
             };
         }
@@ -4063,6 +4139,20 @@ impl Parser<'_> {
         }
     }
 
+    fn parse_corresponding(&mut self) -> PResult<Corresponding> {
+        let corresponding = if self.parse_keyword(Keyword::CORRESPONDING) {
+            let column_list = if self.parse_keyword(Keyword::BY) {
+                Some(self.parse_parenthesized_column_list(IsOptional::Mandatory)?)
+            } else {
+                None
+            };
+            Corresponding::with_column_list(column_list)
+        } else {
+            Corresponding::none()
+        };
+        Ok(corresponding)
+    }
+
     /// Parse a restricted `SELECT` statement (no CTEs / `UNION` / `ORDER BY`),
     /// assuming the initial `SELECT` was already consumed
     pub fn parse_select(&mut self) -> PResult<Select> {
diff --git a/src/sqlparser/tests/sqlparser_common.rs b/src/sqlparser/tests/sqlparser_common.rs
index 16d0d94b601d5..049cf79482032 100644
--- a/src/sqlparser/tests/sqlparser_common.rs
+++ b/src/sqlparser/tests/sqlparser_common.rs
@@ -345,6 +345,7 @@ fn parse_select_count_wildcard() {
     let select = verified_only_select(sql);
     assert_eq!(
         &Expr::Function(Function {
+            scalar_as_agg: false,
             name: ObjectName(vec![Ident::new_unchecked("COUNT")]),
             args: vec![FunctionArg::Unnamed(FunctionArgExpr::Wildcard(None))],
             variadic: false,
@@ -364,6 +365,7 @@ fn parse_select_count_distinct() {
     let select = verified_only_select(sql);
     assert_eq!(
         &Expr::Function(Function {
+            scalar_as_agg: false,
             name: ObjectName(vec![Ident::new_unchecked("COUNT")]),
             args: vec![FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::UnaryOp {
                 op: UnaryOperator::Plus,
@@ -1162,6 +1164,7 @@ fn parse_select_having() {
     assert_eq!(
         Some(Expr::BinaryOp {
             left: Box::new(Expr::Function(Function {
+                scalar_as_agg: false,
                 name: ObjectName(vec![Ident::new_unchecked("COUNT")]),
                 args: vec![FunctionArg::Unnamed(FunctionArgExpr::Wildcard(None))],
                 variadic: false,
@@ -1903,6 +1906,7 @@ fn parse_named_argument_function() {
 
     assert_eq!(
         &Expr::Function(Function {
+            scalar_as_agg: false,
             name: ObjectName(vec![Ident::new_unchecked("FUN")]),
             args: vec![
                 FunctionArg::Named {
@@ -1945,6 +1949,7 @@ fn parse_window_functions() {
     assert_eq!(5, select.projection.len());
     assert_eq!(
         &Expr::Function(Function {
+            scalar_as_agg: false,
             name: ObjectName(vec![Ident::new_unchecked("row_number")]),
             args: vec![],
             variadic: false,
@@ -1979,6 +1984,7 @@ fn parse_aggregate_with_order_by() {
     let select = verified_only_select(sql);
     assert_eq!(
         &Expr::Function(Function {
+            scalar_as_agg: false,
             name: ObjectName(vec![Ident::new_unchecked("STRING_AGG")]),
             args: vec![
                 FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::Identifier(
@@ -2016,6 +2022,7 @@ fn parse_aggregate_with_filter() {
     let select = verified_only_select(sql);
     assert_eq!(
         &Expr::Function(Function {
+            scalar_as_agg: false,
             name: ObjectName(vec![Ident::new_unchecked("sum")]),
             args: vec![FunctionArg::Unnamed(FunctionArgExpr::Expr(
                 Expr::Identifier(Ident::new_unchecked("a"))
@@ -2273,6 +2280,7 @@ fn parse_delimited_identifiers() {
     );
     assert_eq!(
         &Expr::Function(Function {
+            scalar_as_agg: false,
             name: ObjectName(vec![Ident::with_quote_unchecked('"', "myfun")]),
             args: vec![],
             variadic: false,
diff --git a/src/sqlparser/tests/testdata/as_of.yaml b/src/sqlparser/tests/testdata/as_of.yaml
index 2d7c81881988b..903d2a37de748 100644
--- a/src/sqlparser/tests/testdata/as_of.yaml
+++ b/src/sqlparser/tests/testdata/as_of.yaml
@@ -2,7 +2,7 @@
 - input: select * from t1 left join t2 FOR SYSTEM_TIME AS OF PROCTIME() on a1 = a2;
   formatted_sql: SELECT * FROM t1 LEFT JOIN t2 FOR SYSTEM_TIME AS OF PROCTIME() ON a1 = a2
 - input: select * from t1 left join t2 FOR SYSTEM_TIME AS OF NOW() on a1 = a2;
-  formatted_sql: SELECT * FROM t1 LEFT JOIN t2 FOR SYSTEM_TIME AS OF PROCTIME() ON a1 = a2
+  formatted_sql: SELECT * FROM t1 LEFT JOIN t2 FOR SYSTEM_TIME AS OF NOW() - 0 SECOND ON a1 = a2
 - input: select * from t1 left join t2 FOR SYSTEM_TIME AS OF 1 on a1 = a2;
   formatted_sql: SELECT * FROM t1 LEFT JOIN t2 FOR SYSTEM_TIME AS OF 1 ON a1 = a2
 - input: select * from t1 left join t2 FOR SYSTEM_TIME AS OF 'string' on a1 = a2;
diff --git a/src/sqlparser/tests/testdata/lambda.yaml b/src/sqlparser/tests/testdata/lambda.yaml
index b61286d56dc93..ae3f650d73d44 100644
--- a/src/sqlparser/tests/testdata/lambda.yaml
+++ b/src/sqlparser/tests/testdata/lambda.yaml
@@ -1,10 +1,10 @@
 # This file is automatically generated by `src/sqlparser/tests/parser_test.rs`.
 - input: select array_transform(array[1,2,3], |x| x * 2)
   formatted_sql: SELECT array_transform(ARRAY[1, 2, 3], |x| x * 2)
-  formatted_ast: 'Query(Query { with: None, body: Select(Select { distinct: All, projection: [UnnamedExpr(Function(Function { name: ObjectName([Ident { value: "array_transform", quote_style: None }]), args: [Unnamed(Expr(Array(Array { elem: [Value(Number("1")), Value(Number("2")), Value(Number("3"))], named: true }))), Unnamed(Expr(LambdaFunction { args: [Ident { value: "x", quote_style: None }], body: BinaryOp { left: Identifier(Ident { value: "x", quote_style: None }), op: Multiply, right: Value(Number("2")) } }))], variadic: false, over: None, distinct: false, order_by: [], filter: None, within_group: None }))], from: [], lateral_views: [], selection: None, group_by: [], having: None }), order_by: [], limit: None, offset: None, fetch: None })'
+  formatted_ast: 'Query(Query { with: None, body: Select(Select { distinct: All, projection: [UnnamedExpr(Function(Function { scalar_as_agg: false, name: ObjectName([Ident { value: "array_transform", quote_style: None }]), args: [Unnamed(Expr(Array(Array { elem: [Value(Number("1")), Value(Number("2")), Value(Number("3"))], named: true }))), Unnamed(Expr(LambdaFunction { args: [Ident { value: "x", quote_style: None }], body: BinaryOp { left: Identifier(Ident { value: "x", quote_style: None }), op: Multiply, right: Value(Number("2")) } }))], variadic: false, over: None, distinct: false, order_by: [], filter: None, within_group: None }))], from: [], lateral_views: [], selection: None, group_by: [], having: None }), order_by: [], limit: None, offset: None, fetch: None })'
 - input: select array_transform(array[], |s| case when s ilike 'apple%' then 'apple' when s ilike 'google%' then 'google' else 'unknown' end)
   formatted_sql: SELECT array_transform(ARRAY[], |s| CASE WHEN s ILIKE 'apple%' THEN 'apple' WHEN s ILIKE 'google%' THEN 'google' ELSE 'unknown' END)
-  formatted_ast: 'Query(Query { with: None, body: Select(Select { distinct: All, projection: [UnnamedExpr(Function(Function { name: ObjectName([Ident { value: "array_transform", quote_style: None }]), args: [Unnamed(Expr(Array(Array { elem: [], named: true }))), Unnamed(Expr(LambdaFunction { args: [Ident { value: "s", quote_style: None }], body: Case { operand: None, conditions: [ILike { negated: false, expr: Identifier(Ident { value: "s", quote_style: None }), pattern: Value(SingleQuotedString("apple%")), escape_char: None }, ILike { negated: false, expr: Identifier(Ident { value: "s", quote_style: None }), pattern: Value(SingleQuotedString("google%")), escape_char: None }], results: [Value(SingleQuotedString("apple")), Value(SingleQuotedString("google"))], else_result: Some(Value(SingleQuotedString("unknown"))) } }))], variadic: false, over: None, distinct: false, order_by: [], filter: None, within_group: None }))], from: [], lateral_views: [], selection: None, group_by: [], having: None }), order_by: [], limit: None, offset: None, fetch: None })'
+  formatted_ast: 'Query(Query { with: None, body: Select(Select { distinct: All, projection: [UnnamedExpr(Function(Function { scalar_as_agg: false, name: ObjectName([Ident { value: "array_transform", quote_style: None }]), args: [Unnamed(Expr(Array(Array { elem: [], named: true }))), Unnamed(Expr(LambdaFunction { args: [Ident { value: "s", quote_style: None }], body: Case { operand: None, conditions: [ILike { negated: false, expr: Identifier(Ident { value: "s", quote_style: None }), pattern: Value(SingleQuotedString("apple%")), escape_char: None }, ILike { negated: false, expr: Identifier(Ident { value: "s", quote_style: None }), pattern: Value(SingleQuotedString("google%")), escape_char: None }], results: [Value(SingleQuotedString("apple")), Value(SingleQuotedString("google"))], else_result: Some(Value(SingleQuotedString("unknown"))) } }))], variadic: false, over: None, distinct: false, order_by: [], filter: None, within_group: None }))], from: [], lateral_views: [], selection: None, group_by: [], having: None }), order_by: [], limit: None, offset: None, fetch: None })'
 - input: select array_transform(array[], |x, y| x + y * 2)
   formatted_sql: SELECT array_transform(ARRAY[], |x, y| x + y * 2)
-  formatted_ast: 'Query(Query { with: None, body: Select(Select { distinct: All, projection: [UnnamedExpr(Function(Function { name: ObjectName([Ident { value: "array_transform", quote_style: None }]), args: [Unnamed(Expr(Array(Array { elem: [], named: true }))), Unnamed(Expr(LambdaFunction { args: [Ident { value: "x", quote_style: None }, Ident { value: "y", quote_style: None }], body: BinaryOp { left: Identifier(Ident { value: "x", quote_style: None }), op: Plus, right: BinaryOp { left: Identifier(Ident { value: "y", quote_style: None }), op: Multiply, right: Value(Number("2")) } } }))], variadic: false, over: None, distinct: false, order_by: [], filter: None, within_group: None }))], from: [], lateral_views: [], selection: None, group_by: [], having: None }), order_by: [], limit: None, offset: None, fetch: None })'
+  formatted_ast: 'Query(Query { with: None, body: Select(Select { distinct: All, projection: [UnnamedExpr(Function(Function { scalar_as_agg: false, name: ObjectName([Ident { value: "array_transform", quote_style: None }]), args: [Unnamed(Expr(Array(Array { elem: [], named: true }))), Unnamed(Expr(LambdaFunction { args: [Ident { value: "x", quote_style: None }, Ident { value: "y", quote_style: None }], body: BinaryOp { left: Identifier(Ident { value: "x", quote_style: None }), op: Plus, right: BinaryOp { left: Identifier(Ident { value: "y", quote_style: None }), op: Multiply, right: Value(Number("2")) } } }))], variadic: false, over: None, distinct: false, order_by: [], filter: None, within_group: None }))], from: [], lateral_views: [], selection: None, group_by: [], having: None }), order_by: [], limit: None, offset: None, fetch: None })'
diff --git a/src/sqlparser/tests/testdata/qualified_operator.yaml b/src/sqlparser/tests/testdata/qualified_operator.yaml
index 59f3efdabbdbb..83f8e885989c7 100644
--- a/src/sqlparser/tests/testdata/qualified_operator.yaml
+++ b/src/sqlparser/tests/testdata/qualified_operator.yaml
@@ -19,10 +19,10 @@
   formatted_ast: 'Query(Query { with: None, body: Select(Select { distinct: All, projection: [UnnamedExpr(Identifier(Ident { value: "operator", quote_style: None }))], from: [], lateral_views: [], selection: None, group_by: [], having: None }), order_by: [], limit: None, offset: None, fetch: None })'
 - input: select "operator"(foo.bar);
   formatted_sql: SELECT "operator"(foo.bar)
-  formatted_ast: 'Query(Query { with: None, body: Select(Select { distinct: All, projection: [UnnamedExpr(Function(Function { name: ObjectName([Ident { value: "operator", quote_style: Some(''"'') }]), args: [Unnamed(Expr(CompoundIdentifier([Ident { value: "foo", quote_style: None }, Ident { value: "bar", quote_style: None }])))], variadic: false, over: None, distinct: false, order_by: [], filter: None, within_group: None }))], from: [], lateral_views: [], selection: None, group_by: [], having: None }), order_by: [], limit: None, offset: None, fetch: None })'
+  formatted_ast: 'Query(Query { with: None, body: Select(Select { distinct: All, projection: [UnnamedExpr(Function(Function { scalar_as_agg: false, name: ObjectName([Ident { value: "operator", quote_style: Some(''"'') }]), args: [Unnamed(Expr(CompoundIdentifier([Ident { value: "foo", quote_style: None }, Ident { value: "bar", quote_style: None }])))], variadic: false, over: None, distinct: false, order_by: [], filter: None, within_group: None }))], from: [], lateral_views: [], selection: None, group_by: [], having: None }), order_by: [], limit: None, offset: None, fetch: None })'
 - input: select operator operator(+) operator(+) "operator"(9) operator from operator;
   formatted_sql: SELECT operator OPERATOR(+) OPERATOR(+) "operator"(9) AS operator FROM operator
-  formatted_ast: 'Query(Query { with: None, body: Select(Select { distinct: All, projection: [ExprWithAlias { expr: BinaryOp { left: Identifier(Ident { value: "operator", quote_style: None }), op: PGQualified(QualifiedOperator { schema: None, name: "+" }), right: UnaryOp { op: PGQualified(QualifiedOperator { schema: None, name: "+" }), expr: Function(Function { name: ObjectName([Ident { value: "operator", quote_style: Some(''"'') }]), args: [Unnamed(Expr(Value(Number("9"))))], variadic: false, over: None, distinct: false, order_by: [], filter: None, within_group: None }) } }, alias: Ident { value: "operator", quote_style: None } }], from: [TableWithJoins { relation: Table { name: ObjectName([Ident { value: "operator", quote_style: None }]), alias: None, as_of: None }, joins: [] }], lateral_views: [], selection: None, group_by: [], having: None }), order_by: [], limit: None, offset: None, fetch: None })'
+  formatted_ast: 'Query(Query { with: None, body: Select(Select { distinct: All, projection: [ExprWithAlias { expr: BinaryOp { left: Identifier(Ident { value: "operator", quote_style: None }), op: PGQualified(QualifiedOperator { schema: None, name: "+" }), right: UnaryOp { op: PGQualified(QualifiedOperator { schema: None, name: "+" }), expr: Function(Function { scalar_as_agg: false, name: ObjectName([Ident { value: "operator", quote_style: Some(''"'') }]), args: [Unnamed(Expr(Value(Number("9"))))], variadic: false, over: None, distinct: false, order_by: [], filter: None, within_group: None }) } }, alias: Ident { value: "operator", quote_style: None } }], from: [TableWithJoins { relation: Table { name: ObjectName([Ident { value: "operator", quote_style: None }]), alias: None, as_of: None }, joins: [] }], lateral_views: [], selection: None, group_by: [], having: None }), order_by: [], limit: None, offset: None, fetch: None })'
 - input: select 3 operator(-) 2 - 1;
   formatted_sql: SELECT 3 OPERATOR(-) 2 - 1
   formatted_ast: 'Query(Query { with: None, body: Select(Select { distinct: All, projection: [UnnamedExpr(BinaryOp { left: Value(Number("3")), op: PGQualified(QualifiedOperator { schema: None, name: "-" }), right: BinaryOp { left: Value(Number("2")), op: Minus, right: Value(Number("1")) } })], from: [], lateral_views: [], selection: None, group_by: [], having: None }), order_by: [], limit: None, offset: None, fetch: None })'
diff --git a/src/sqlparser/tests/testdata/select.yaml b/src/sqlparser/tests/testdata/select.yaml
index f034aa8a58c9f..06c12a3e7d554 100644
--- a/src/sqlparser/tests/testdata/select.yaml
+++ b/src/sqlparser/tests/testdata/select.yaml
@@ -1,7 +1,7 @@
 # This file is automatically generated by `src/sqlparser/tests/parser_test.rs`.
 - input: SELECT sqrt(id) FROM foo
   formatted_sql: SELECT sqrt(id) FROM foo
-  formatted_ast: 'Query(Query { with: None, body: Select(Select { distinct: All, projection: [UnnamedExpr(Function(Function { name: ObjectName([Ident { value: "sqrt", quote_style: None }]), args: [Unnamed(Expr(Identifier(Ident { value: "id", quote_style: None })))], variadic: false, over: None, distinct: false, order_by: [], filter: None, within_group: None }))], from: [TableWithJoins { relation: Table { name: ObjectName([Ident { value: "foo", quote_style: None }]), alias: None, as_of: None }, joins: [] }], lateral_views: [], selection: None, group_by: [], having: None }), order_by: [], limit: None, offset: None, fetch: None })'
+  formatted_ast: 'Query(Query { with: None, body: Select(Select { distinct: All, projection: [UnnamedExpr(Function(Function { scalar_as_agg: false, name: ObjectName([Ident { value: "sqrt", quote_style: None }]), args: [Unnamed(Expr(Identifier(Ident { value: "id", quote_style: None })))], variadic: false, over: None, distinct: false, order_by: [], filter: None, within_group: None }))], from: [TableWithJoins { relation: Table { name: ObjectName([Ident { value: "foo", quote_style: None }]), alias: None, as_of: None }, joins: [] }], lateral_views: [], selection: None, group_by: [], having: None }), order_by: [], limit: None, offset: None, fetch: None })'
 - input: SELECT INT '1'
   formatted_sql: SELECT INT '1'
 - input: SELECT (foo).v1.v2 FROM foo
@@ -99,7 +99,7 @@
   formatted_ast: 'Query(Query { with: None, body: Select(Select { distinct: All, projection: [UnnamedExpr(AtTimeZone { timestamp: TypedString { data_type: Timestamp(true), value: "2022-10-01 12:00:00Z" }, time_zone: Identifier(Ident { value: "zone", quote_style: None }) })], from: [], lateral_views: [], selection: None, group_by: [], having: None }), order_by: [], limit: None, offset: None, fetch: None })'
 - input: SELECT now() + INTERVAL '14 days' AT TIME ZONE 'UTC'; -- https://www.postgresql.org/message-id/CADT4RqBPdbsZW7HS1jJP319TMRHs1hzUiP=iRJYR6UqgHCrgNQ@mail.gmail.com
   formatted_sql: SELECT now() + INTERVAL '14 days' AT TIME ZONE 'UTC'
-  formatted_ast: 'Query(Query { with: None, body: Select(Select { distinct: All, projection: [UnnamedExpr(BinaryOp { left: Function(Function { name: ObjectName([Ident { value: "now", quote_style: None }]), args: [], variadic: false, over: None, distinct: false, order_by: [], filter: None, within_group: None }), op: Plus, right: AtTimeZone { timestamp: Value(Interval { value: "14 days", leading_field: None, leading_precision: None, last_field: None, fractional_seconds_precision: None }), time_zone: Value(SingleQuotedString("UTC")) } })], from: [], lateral_views: [], selection: None, group_by: [], having: None }), order_by: [], limit: None, offset: None, fetch: None })'
+  formatted_ast: 'Query(Query { with: None, body: Select(Select { distinct: All, projection: [UnnamedExpr(BinaryOp { left: Function(Function { scalar_as_agg: false, name: ObjectName([Ident { value: "now", quote_style: None }]), args: [], variadic: false, over: None, distinct: false, order_by: [], filter: None, within_group: None }), op: Plus, right: AtTimeZone { timestamp: Value(Interval { value: "14 days", leading_field: None, leading_precision: None, last_field: None, fractional_seconds_precision: None }), time_zone: Value(SingleQuotedString("UTC")) } })], from: [], lateral_views: [], selection: None, group_by: [], having: None }), order_by: [], limit: None, offset: None, fetch: None })'
 - input: SELECT c FROM t WHERE c >= '2019-03-27T22:00:00.000Z'::timestamp AT TIME ZONE 'Europe/Brussels'; -- https://github.com/sqlparser-rs/sqlparser-rs/issues/1266
   formatted_sql: SELECT c FROM t WHERE c >= CAST('2019-03-27T22:00:00.000Z' AS TIMESTAMP) AT TIME ZONE 'Europe/Brussels'
   formatted_ast: 'Query(Query { with: None, body: Select(Select { distinct: All, projection: [UnnamedExpr(Identifier(Ident { value: "c", quote_style: None }))], from: [TableWithJoins { relation: Table { name: ObjectName([Ident { value: "t", quote_style: None }]), alias: None, as_of: None }, joins: [] }], lateral_views: [], selection: Some(BinaryOp { left: Identifier(Ident { value: "c", quote_style: None }), op: GtEq, right: AtTimeZone { timestamp: Cast { expr: Value(SingleQuotedString("2019-03-27T22:00:00.000Z")), data_type: Timestamp(false) }, time_zone: Value(SingleQuotedString("Europe/Brussels")) } }), group_by: [], having: None }), order_by: [], limit: None, offset: None, fetch: None })'
@@ -173,7 +173,7 @@
   formatted_ast: 'Query(Query { with: None, body: Select(Select { distinct: All, projection: [UnnamedExpr(Identifier(Ident { value: "id1", quote_style: None })), UnnamedExpr(Identifier(Ident { value: "a1", quote_style: None })), UnnamedExpr(Identifier(Ident { value: "id2", quote_style: None })), UnnamedExpr(Identifier(Ident { value: "a2", quote_style: None }))], from: [TableWithJoins { relation: Table { name: ObjectName([Ident { value: "stream", quote_style: None }]), alias: Some(TableAlias { name: Ident { value: "S", quote_style: None }, columns: [] }), as_of: None }, joins: [Join { relation: Table { name: ObjectName([Ident { value: "version", quote_style: None }]), alias: Some(TableAlias { name: Ident { value: "V", quote_style: None }, columns: [] }), as_of: Some(ProcessTime) }, join_operator: Inner(On(BinaryOp { left: Identifier(Ident { value: "id1", quote_style: None }), op: Eq, right: Identifier(Ident { value: "id2", quote_style: None }) })) }] }], lateral_views: [], selection: None, group_by: [], having: None }), order_by: [], limit: None, offset: None, fetch: None })'
 - input: select percentile_cont(0.3) within group (order by x desc) from unnest(array[1,2,4,5,10]) as x
   formatted_sql: SELECT percentile_cont(0.3) FROM unnest(ARRAY[1, 2, 4, 5, 10]) AS x
-  formatted_ast: 'Query(Query { with: None, body: Select(Select { distinct: All, projection: [UnnamedExpr(Function(Function { name: ObjectName([Ident { value: "percentile_cont", quote_style: None }]), args: [Unnamed(Expr(Value(Number("0.3"))))], variadic: false, over: None, distinct: false, order_by: [], filter: None, within_group: Some(OrderByExpr { expr: Identifier(Ident { value: "x", quote_style: None }), asc: Some(false), nulls_first: None }) }))], from: [TableWithJoins { relation: TableFunction { name: ObjectName([Ident { value: "unnest", quote_style: None }]), alias: Some(TableAlias { name: Ident { value: "x", quote_style: None }, columns: [] }), args: [Unnamed(Expr(Array(Array { elem: [Value(Number("1")), Value(Number("2")), Value(Number("4")), Value(Number("5")), Value(Number("10"))], named: true })))], with_ordinality: false }, joins: [] }], lateral_views: [], selection: None, group_by: [], having: None }), order_by: [], limit: None, offset: None, fetch: None })'
+  formatted_ast: 'Query(Query { with: None, body: Select(Select { distinct: All, projection: [UnnamedExpr(Function(Function { scalar_as_agg: false, name: ObjectName([Ident { value: "percentile_cont", quote_style: None }]), args: [Unnamed(Expr(Value(Number("0.3"))))], variadic: false, over: None, distinct: false, order_by: [], filter: None, within_group: Some(OrderByExpr { expr: Identifier(Ident { value: "x", quote_style: None }), asc: Some(false), nulls_first: None }) }))], from: [TableWithJoins { relation: TableFunction { name: ObjectName([Ident { value: "unnest", quote_style: None }]), alias: Some(TableAlias { name: Ident { value: "x", quote_style: None }, columns: [] }), args: [Unnamed(Expr(Array(Array { elem: [Value(Number("1")), Value(Number("2")), Value(Number("4")), Value(Number("5")), Value(Number("10"))], named: true })))], with_ordinality: false }, joins: [] }], lateral_views: [], selection: None, group_by: [], having: None }), order_by: [], limit: None, offset: None, fetch: None })'
 - input: select percentile_cont(0.3) within group (order by x, y desc) from t
   error_msg: |-
     sql parser error: expected ), found: ,
diff --git a/src/sqlparser/tests/testdata/subquery.yaml b/src/sqlparser/tests/testdata/subquery.yaml
index 71fba982586e0..d9c2d0a24ee7f 100644
--- a/src/sqlparser/tests/testdata/subquery.yaml
+++ b/src/sqlparser/tests/testdata/subquery.yaml
@@ -23,4 +23,4 @@
   formatted_ast: 'Query(Query { with: None, body: Select(Select { distinct: All, projection: [UnnamedExpr(BinaryOp { left: Row([Value(Number("1")), Value(Boolean(true))]), op: Lt, right: Subquery(Query { with: None, body: Select(Select { distinct: All, projection: [UnnamedExpr(Value(Number("2"))), UnnamedExpr(Value(Boolean(false)))], from: [], lateral_views: [], selection: None, group_by: [], having: None }), order_by: [], limit: None, offset: None, fetch: None }) })], from: [], lateral_views: [], selection: None, group_by: [], having: None }), order_by: [], limit: None, offset: None, fetch: None })'
 - input: select array(select 2 union select 3);
   formatted_sql: SELECT ARRAY (SELECT 2 UNION SELECT 3)
-  formatted_ast: 'Query(Query { with: None, body: Select(Select { distinct: All, projection: [UnnamedExpr(ArraySubquery(Query { with: None, body: SetOperation { op: Union, all: false, left: Select(Select { distinct: All, projection: [UnnamedExpr(Value(Number("2")))], from: [], lateral_views: [], selection: None, group_by: [], having: None }), right: Select(Select { distinct: All, projection: [UnnamedExpr(Value(Number("3")))], from: [], lateral_views: [], selection: None, group_by: [], having: None }) }, order_by: [], limit: None, offset: None, fetch: None }))], from: [], lateral_views: [], selection: None, group_by: [], having: None }), order_by: [], limit: None, offset: None, fetch: None })'
+  formatted_ast: 'Query(Query { with: None, body: Select(Select { distinct: All, projection: [UnnamedExpr(ArraySubquery(Query { with: None, body: SetOperation { op: Union, all: false, corresponding: Corresponding { corresponding: false, column_list: None }, left: Select(Select { distinct: All, projection: [UnnamedExpr(Value(Number("2")))], from: [], lateral_views: [], selection: None, group_by: [], having: None }), right: Select(Select { distinct: All, projection: [UnnamedExpr(Value(Number("3")))], from: [], lateral_views: [], selection: None, group_by: [], having: None }) }, order_by: [], limit: None, offset: None, fetch: None }))], from: [], lateral_views: [], selection: None, group_by: [], having: None }), order_by: [], limit: None, offset: None, fetch: None })'
diff --git a/src/sqlparser/tests/testdata/union.yaml b/src/sqlparser/tests/testdata/union.yaml
new file mode 100644
index 0000000000000..5208a8b0c5b6a
--- /dev/null
+++ b/src/sqlparser/tests/testdata/union.yaml
@@ -0,0 +1,7 @@
+# This file is automatically generated by `src/sqlparser/tests/parser_test.rs`.
+- input: select * from t1 union all corresponding by (a, b) select v1, v2 from t2;
+  formatted_sql: SELECT * FROM t1 UNION ALL CORRESPONDING BY (a, b) SELECT v1, v2 FROM t2
+  formatted_ast: 'Query(Query { with: None, body: SetOperation { op: Union, all: true, corresponding: Corresponding { corresponding: true, column_list: Some([Ident { value: "a", quote_style: None }, Ident { value: "b", quote_style: None }]) }, left: Select(Select { distinct: All, projection: [Wildcard(None)], from: [TableWithJoins { relation: Table { name: ObjectName([Ident { value: "t1", quote_style: None }]), alias: None, as_of: None }, joins: [] }], lateral_views: [], selection: None, group_by: [], having: None }), right: Select(Select { distinct: All, projection: [UnnamedExpr(Identifier(Ident { value: "v1", quote_style: None })), UnnamedExpr(Identifier(Ident { value: "v2", quote_style: None }))], from: [TableWithJoins { relation: Table { name: ObjectName([Ident { value: "t2", quote_style: None }]), alias: None, as_of: None }, joins: [] }], lateral_views: [], selection: None, group_by: [], having: None }) }, order_by: [], limit: None, offset: None, fetch: None })'
+- input: select * from t1 union select v1, v2 from t2;
+  formatted_sql: SELECT * FROM t1 UNION SELECT v1, v2 FROM t2
+  formatted_ast: 'Query(Query { with: None, body: SetOperation { op: Union, all: false, corresponding: Corresponding { corresponding: false, column_list: None }, left: Select(Select { distinct: All, projection: [Wildcard(None)], from: [TableWithJoins { relation: Table { name: ObjectName([Ident { value: "t1", quote_style: None }]), alias: None, as_of: None }, joins: [] }], lateral_views: [], selection: None, group_by: [], having: None }), right: Select(Select { distinct: All, projection: [UnnamedExpr(Identifier(Ident { value: "v1", quote_style: None })), UnnamedExpr(Identifier(Ident { value: "v2", quote_style: None }))], from: [TableWithJoins { relation: Table { name: ObjectName([Ident { value: "t2", quote_style: None }]), alias: None, as_of: None }, joins: [] }], lateral_views: [], selection: None, group_by: [], having: None }) }, order_by: [], limit: None, offset: None, fetch: None })'
diff --git a/src/storage/Cargo.toml b/src/storage/Cargo.toml
index 0dc4bd8b088f1..b49a625111d37 100644
--- a/src/storage/Cargo.toml
+++ b/src/storage/Cargo.toml
@@ -36,6 +36,7 @@ libc = "0.2"
 lz4 = "1.25.0"
 memcomparable = "0.2"
 metrics-prometheus = "0.7"
+moka = { version = "0.12", features = ["future", "sync"] }
 more-asserts = "0.3"
 num-integer = "0.1"
 parking_lot = { workspace = true }
@@ -95,7 +96,6 @@ workspace-hack = { path = "../workspace-hack" }
 bincode = "1"
 criterion = { workspace = true, features = ["async_futures", "async_tokio"] }
 expect-test = "1"
-moka = { version = "0.12", features = ["future"] }
 risingwave_hummock_sdk = { workspace = true }
 risingwave_test_runner = { workspace = true }
 uuid = { version = "1", features = ["v4"] }
diff --git a/src/storage/backup/integration_tests/common.sh b/src/storage/backup/integration_tests/common.sh
index d28ab1b9a9827..d72b47686953b 100644
--- a/src/storage/backup/integration_tests/common.sh
+++ b/src/storage/backup/integration_tests/common.sh
@@ -111,7 +111,7 @@ function backup() {
 function delete_snapshot() {
   local snapshot_id
   snapshot_id=$1
-  ${BACKUP_TEST_RW_ALL_IN_ONE} risectl meta delete-meta-snapshots "${snapshot_id}"
+  ${BACKUP_TEST_RW_ALL_IN_ONE} risectl meta delete-meta-snapshots --snapshot-ids "${snapshot_id}"
 }
 
 function restore() {
diff --git a/src/storage/backup/src/lib.rs b/src/storage/backup/src/lib.rs
index 6b569277c54dd..6423f9c4dd87b 100644
--- a/src/storage/backup/src/lib.rs
+++ b/src/storage/backup/src/lib.rs
@@ -21,7 +21,6 @@
 #![feature(map_try_insert)]
 #![feature(hash_extract_if)]
 #![feature(btree_extract_if)]
-#![feature(lazy_cell)]
 #![feature(let_chains)]
 #![feature(error_generic_member_access)]
 #![cfg_attr(coverage, feature(coverage_attribute))]
@@ -60,7 +59,7 @@ pub struct MetaSnapshotMetadata {
     #[serde(default)]
     pub format_version: u32,
     pub remarks: Option<String>,
-    #[serde(with = "table_id_key_map")]
+    #[serde(default, with = "table_id_key_map")]
     pub state_table_info: HashMap<TableId, PbStateTableInfo>,
     pub rw_version: Option<String>,
 }
diff --git a/src/storage/backup/src/meta_snapshot_v1.rs b/src/storage/backup/src/meta_snapshot_v1.rs
index f94e3e1446eb1..9b4145ea25632 100644
--- a/src/storage/backup/src/meta_snapshot_v1.rs
+++ b/src/storage/backup/src/meta_snapshot_v1.rs
@@ -20,9 +20,9 @@ use itertools::Itertools;
 use risingwave_common::util::iter_util::ZipEqFast;
 use risingwave_hummock_sdk::version::HummockVersion;
 use risingwave_pb::catalog::{
-    Connection, Database, Function, Index, Schema, Sink, Source, Subscription, Table, View,
+    Connection, Database, Function, Index, Schema, Secret, Sink, Source, Subscription, Table, View,
 };
-use risingwave_pb::hummock::{CompactionGroup, HummockVersionStats};
+use risingwave_pb::hummock::{CompactionGroup, HummockVersionStats, PbHummockVersion};
 use risingwave_pb::meta::{SystemParams, TableFragments};
 use risingwave_pb::user::UserInfo;
 
@@ -72,6 +72,10 @@ impl Display for ClusterMetadata {
         writeln!(f, "{:#?}", self.system_param)?;
         writeln!(f, "cluster_id:")?;
         writeln!(f, "{:#?}", self.cluster_id)?;
+        writeln!(f, "subscription:")?;
+        writeln!(f, "{:#?}", self.subscription)?;
+        writeln!(f, "secret:")?;
+        writeln!(f, "{:#?}", self.secret)?;
         Ok(())
     }
 }
@@ -121,6 +125,7 @@ pub struct ClusterMetadata {
     pub system_param: SystemParams,
     pub cluster_id: String,
     pub subscription: Vec<Subscription>,
+    pub secret: Vec<Secret>,
 }
 
 impl ClusterMetadata {
@@ -129,7 +134,7 @@ impl ClusterMetadata {
         let default_cf_values = self.default_cf.values().collect_vec();
         Self::encode_prost_message_list(&default_cf_keys, buf);
         Self::encode_prost_message_list(&default_cf_values, buf);
-        Self::encode_prost_message(&self.hummock_version.to_protobuf(), buf);
+        Self::encode_prost_message(&PbHummockVersion::from(&self.hummock_version), buf);
         Self::encode_prost_message(&self.version_stats, buf);
         Self::encode_prost_message_list(&self.compaction_groups.iter().collect_vec(), buf);
         Self::encode_prost_message_list(&self.table_fragments.iter().collect_vec(), buf);
@@ -146,6 +151,7 @@ impl ClusterMetadata {
         Self::encode_prost_message(&self.system_param, buf);
         Self::encode_prost_message(&self.cluster_id, buf);
         Self::encode_prost_message_list(&self.subscription.iter().collect_vec(), buf);
+        Self::encode_prost_message_list(&self.secret.iter().collect_vec(), buf);
         Ok(())
     }
 
@@ -175,6 +181,8 @@ impl ClusterMetadata {
         let cluster_id: String = Self::decode_prost_message(&mut buf)?;
         let subscription: Vec<Subscription> =
             Self::try_decode_prost_message_list(&mut buf).unwrap_or_else(|| Ok(vec![]))?;
+        let secret: Vec<Secret> =
+            Self::try_decode_prost_message_list(&mut buf).unwrap_or_else(|| Ok(vec![]))?;
 
         Ok(Self {
             default_cf,
@@ -195,6 +203,7 @@ impl ClusterMetadata {
             system_param,
             cluster_id,
             subscription,
+            secret,
         })
     }
 
diff --git a/src/storage/benches/bench_block_iter.rs b/src/storage/benches/bench_block_iter.rs
index f58499e07282f..a0ea7cfd844d1 100644
--- a/src/storage/benches/bench_block_iter.rs
+++ b/src/storage/benches/bench_block_iter.rs
@@ -12,7 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#![feature(lazy_cell)]
 use std::sync::LazyLock;
 
 use bytes::{BufMut, Bytes, BytesMut};
diff --git a/src/storage/benches/bench_compactor.rs b/src/storage/benches/bench_compactor.rs
index 08da1438709c5..e5f929a312a88 100644
--- a/src/storage/benches/bench_compactor.rs
+++ b/src/storage/benches/bench_compactor.rs
@@ -27,9 +27,11 @@ use risingwave_common::util::value_encoding::column_aware_row_encoding::ColumnAw
 use risingwave_common::util::value_encoding::ValueRowSerializer;
 use risingwave_hummock_sdk::key::FullKey;
 use risingwave_hummock_sdk::key_range::KeyRange;
+use risingwave_hummock_sdk::sstable_info::SstableInfo;
 use risingwave_object_store::object::object_metrics::ObjectStoreMetrics;
 use risingwave_object_store::object::{InMemObjectStore, ObjectStore, ObjectStoreImpl};
-use risingwave_pb::hummock::{compact_task, SstableInfo, TableSchema};
+use risingwave_pb::hummock::compact_task::PbTaskType;
+use risingwave_pb::hummock::PbTableSchema;
 use risingwave_storage::hummock::compactor::compactor_runner::compact_and_build_sst;
 use risingwave_storage::hummock::compactor::{
     ConcatSstableIterator, DummyCompactionFilter, TaskConfig, TaskProgress,
@@ -280,7 +282,7 @@ async fn compact<I: HummockIterator<Direction = Forward>>(
         gc_delete_keys: false,
         watermark: 0,
         stats_target_table_ids: None,
-        task_type: compact_task::TaskType::Dynamic,
+        task_type: PbTaskType::Dynamic,
         use_block_based_filter: true,
         ..Default::default()
     });
@@ -432,7 +434,7 @@ fn bench_drop_column_compaction_impl(c: &mut Criterion, column_num: usize) {
         gc_delete_keys: false,
         watermark: 0,
         stats_target_table_ids: None,
-        task_type: compact_task::TaskType::Dynamic,
+        task_type: PbTaskType::Dynamic,
         use_block_based_filter: true,
         table_schemas: vec![].into_iter().collect(),
         ..Default::default()
@@ -441,13 +443,13 @@ fn bench_drop_column_compaction_impl(c: &mut Criterion, column_num: usize) {
     let mut task_config_schema = task_config_no_schema.clone();
     task_config_schema.table_schemas.insert(
         10,
-        TableSchema {
+        PbTableSchema {
             column_ids: (0..column_num as i32).collect(),
         },
     );
     task_config_schema.table_schemas.insert(
         11,
-        TableSchema {
+        PbTableSchema {
             column_ids: (0..column_num as i32).collect(),
         },
     );
@@ -455,13 +457,13 @@ fn bench_drop_column_compaction_impl(c: &mut Criterion, column_num: usize) {
     let mut task_config_schema_cause_drop = task_config_no_schema.clone();
     task_config_schema_cause_drop.table_schemas.insert(
         10,
-        TableSchema {
+        PbTableSchema {
             column_ids: (0..column_num as i32 / 2).collect(),
         },
     );
     task_config_schema_cause_drop.table_schemas.insert(
         11,
-        TableSchema {
+        PbTableSchema {
             column_ids: (0..column_num as i32 / 2).collect(),
         },
     );
diff --git a/src/storage/benches/bench_multi_builder.rs b/src/storage/benches/bench_multi_builder.rs
index 08d744189a1d8..730534a81831a 100644
--- a/src/storage/benches/bench_multi_builder.rs
+++ b/src/storage/benches/bench_multi_builder.rs
@@ -25,10 +25,10 @@ use rand::random;
 use risingwave_common::catalog::TableId;
 use risingwave_common::config::{MetricLevel, ObjectStoreConfig};
 use risingwave_hummock_sdk::key::{FullKey, UserKey};
+use risingwave_hummock_sdk::sstable_info::SstableInfo;
 use risingwave_object_store::object::{
     InMemObjectStore, ObjectStore, ObjectStoreImpl, S3ObjectStore,
 };
-use risingwave_pb::hummock::SstableInfo;
 use risingwave_storage::hummock::iterator::{ConcatIterator, ConcatIteratorInner, HummockIterator};
 use risingwave_storage::hummock::multi_builder::{CapacitySplitTableBuilder, TableBuilderFactory};
 use risingwave_storage::hummock::value::HummockValue;
diff --git a/src/storage/benches/bench_table_watermarks.rs b/src/storage/benches/bench_table_watermarks.rs
index f1fdf66c2f895..531652065f014 100644
--- a/src/storage/benches/bench_table_watermarks.rs
+++ b/src/storage/benches/bench_table_watermarks.rs
@@ -12,8 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#![feature(lazy_cell)]
-
 use std::collections::hash_map::Entry;
 use std::collections::{HashMap, HashSet, VecDeque};
 use std::sync::{Arc, LazyLock};
@@ -21,7 +19,7 @@ use std::sync::{Arc, LazyLock};
 use bytes::Bytes;
 use criterion::{criterion_group, criterion_main, BatchSize, Criterion};
 use itertools::Itertools;
-use risingwave_common::buffer::{Bitmap, BitmapBuilder};
+use risingwave_common::bitmap::{Bitmap, BitmapBuilder};
 use risingwave_common::catalog::TableId;
 use risingwave_common::hash::VirtualNode;
 use risingwave_common::util::epoch::test_epoch;
diff --git a/src/storage/compactor/Cargo.toml b/src/storage/compactor/Cargo.toml
index f9d943be27964..ed8f421ee6384 100644
--- a/src/storage/compactor/Cargo.toml
+++ b/src/storage/compactor/Cargo.toml
@@ -18,6 +18,7 @@ normal = ["workspace-hack"]
 async-trait = "0.1"
 await-tree = { workspace = true }
 clap = { workspace = true }
+jsonbb = { workspace = true }
 parking_lot = { workspace = true }
 prost = { workspace = true }
 risingwave_common = { workspace = true }
@@ -29,13 +30,13 @@ risingwave_rpc_client = { workspace = true }
 risingwave_storage = { workspace = true }
 serde = { version = "1", features = ["derive"] }
 tokio = { version = "0.2", package = "madsim-tokio", features = [
-    "fs",
-    "rt",
-    "rt-multi-thread",
-    "sync",
-    "macros",
-    "time",
-    "signal",
+  "fs",
+  "rt",
+  "rt-multi-thread",
+  "sync",
+  "macros",
+  "time",
+  "signal",
 ] }
 tonic = { workspace = true }
 tracing = "0.1"
diff --git a/src/storage/compactor/src/compactor_observer/observer_manager.rs b/src/storage/compactor/src/compactor_observer/observer_manager.rs
index 3a37fd6060595..97d78f6c2849f 100644
--- a/src/storage/compactor/src/compactor_observer/observer_manager.rs
+++ b/src/storage/compactor/src/compactor_observer/observer_manager.rs
@@ -16,7 +16,7 @@ use std::collections::HashMap;
 use std::sync::Arc;
 
 use risingwave_common::system_param::local_manager::LocalSystemParamsManagerRef;
-use risingwave_common_service::observer_manager::{ObserverState, SubscribeCompactor};
+use risingwave_common_service::ObserverState;
 use risingwave_pb::catalog::Table;
 use risingwave_pb::meta::relation::RelationInfo;
 use risingwave_pb::meta::subscribe_response::{Info, Operation};
@@ -32,7 +32,9 @@ pub struct CompactorObserverNode {
 }
 
 impl ObserverState for CompactorObserverNode {
-    type SubscribeType = SubscribeCompactor;
+    fn subscribe_type() -> risingwave_pb::meta::SubscribeType {
+        risingwave_pb::meta::SubscribeType::Compactor
+    }
 
     fn handle_notification(&mut self, resp: SubscribeResponse) {
         let Some(info) = resp.info.as_ref() else {
diff --git a/src/storage/compactor/src/lib.rs b/src/storage/compactor/src/lib.rs
index d9fbe5189f43f..22e70ac759aed 100644
--- a/src/storage/compactor/src/lib.rs
+++ b/src/storage/compactor/src/lib.rs
@@ -24,6 +24,7 @@ use risingwave_common::config::{
     AsyncStackTraceOption, CompactorMode, MetricLevel, OverrideConfig,
 };
 use risingwave_common::util::meta_addr::MetaAddressStrategy;
+use risingwave_common::util::tokio_util::sync::CancellationToken;
 
 use crate::server::{compactor_serve, shared_compactor_serve};
 
@@ -108,7 +109,10 @@ impl risingwave_common::opts::Opts for CompactorOpts {
 use std::future::Future;
 use std::pin::Pin;
 
-pub fn start(opts: CompactorOpts) -> Pin<Box<dyn Future<Output = ()> + Send>> {
+pub fn start(
+    opts: CompactorOpts,
+    shutdown: CancellationToken,
+) -> Pin<Box<dyn Future<Output = ()> + Send>> {
     // WARNING: don't change the function signature. Making it `async fn` will cause
     // slow compile in release mode.
     match opts.compactor_mode {
@@ -118,11 +122,7 @@ pub fn start(opts: CompactorOpts) -> Pin<Box<dyn Future<Output = ()> + Send>> {
 
             let listen_addr = opts.listen_addr.parse().unwrap();
 
-            let (join_handle, _shutdown_sender) = shared_compactor_serve(listen_addr, opts).await;
-
-            tracing::info!("Server listening at {}", listen_addr);
-
-            join_handle.await.unwrap();
+            shared_compactor_serve(listen_addr, opts, shutdown).await;
         }),
         None | Some(CompactorMode::Dedicated) => Box::pin(async move {
             tracing::info!("Compactor node options: {:?}", opts);
@@ -140,13 +140,8 @@ pub fn start(opts: CompactorOpts) -> Pin<Box<dyn Future<Output = ()> + Send>> {
                 .parse()
                 .unwrap();
             tracing::info!(" address is {}", advertise_addr);
-            let (join_handle, observer_join_handle, _shutdown_sender) =
-                compactor_serve(listen_addr, advertise_addr, opts).await;
-
-            tracing::info!("Server listening at {}", listen_addr);
 
-            join_handle.await.unwrap();
-            observer_join_handle.abort();
+            compactor_serve(listen_addr, advertise_addr, opts, shutdown).await;
         }),
     }
 }
diff --git a/src/storage/compactor/src/server.rs b/src/storage/compactor/src/server.rs
index 41091e142b838..e139bc201cd48 100644
--- a/src/storage/compactor/src/server.rs
+++ b/src/storage/compactor/src/server.rs
@@ -19,17 +19,17 @@ use std::time::Duration;
 use risingwave_common::config::{
     extract_storage_memory_config, load_config, AsyncStackTraceOption, MetricLevel, RwConfig,
 };
-use risingwave_common::monitor::connection::{RouterExt, TcpConfig};
+use risingwave_common::monitor::{RouterExt, TcpConfig};
 use risingwave_common::system_param::local_manager::LocalSystemParamsManager;
 use risingwave_common::system_param::reader::{SystemParamsRead, SystemParamsReader};
 use risingwave_common::telemetry::manager::TelemetryManager;
 use risingwave_common::telemetry::telemetry_env_enabled;
 use risingwave_common::util::addr::HostAddr;
 use risingwave_common::util::resource_util::memory::system_memory_available_bytes;
+use risingwave_common::util::tokio_util::sync::CancellationToken;
 use risingwave_common::{GIT_SHA, RW_VERSION};
 use risingwave_common_heap_profiling::HeapProfiler;
-use risingwave_common_service::metrics_manager::MetricsManager;
-use risingwave_common_service::observer_manager::ObserverManager;
+use risingwave_common_service::{MetricsManager, ObserverManager};
 use risingwave_object_store::object::build_remote_object_store;
 use risingwave_object_store::object::object_metrics::GLOBAL_OBJECT_STORE_METRICS;
 use risingwave_pb::common::WorkerType;
@@ -52,8 +52,6 @@ use risingwave_storage::monitor::{
 };
 use risingwave_storage::opts::StorageOpts;
 use tokio::sync::mpsc;
-use tokio::sync::oneshot::Sender;
-use tokio::task::JoinHandle;
 use tonic::transport::Endpoint;
 use tracing::info;
 
@@ -174,11 +172,14 @@ pub async fn prepare_start_parameters(
 }
 
 /// Fetches and runs compaction tasks.
+///
+/// Returns when the `shutdown` token is triggered.
 pub async fn compactor_serve(
     listen_addr: SocketAddr,
     advertise_addr: HostAddr,
     opts: CompactorOpts,
-) -> (JoinHandle<()>, JoinHandle<()>, Sender<()>) {
+    shutdown: CancellationToken,
+) {
     let config = load_config(&opts.config_path, &opts);
     info!("Starting compactor node",);
     info!("> config: {:?}", config);
@@ -200,7 +201,6 @@ pub async fn compactor_serve(
     .unwrap();
 
     info!("Assigned compactor id {}", meta_client.worker_id());
-    meta_client.activate(&advertise_addr).await.unwrap();
 
     let hummock_metrics = Arc::new(GLOBAL_HUMMOCK_METRICS.clone());
 
@@ -234,7 +234,7 @@ pub async fn compactor_serve(
 
     // use half of limit because any memory which would hold in meta-cache will be allocate by
     // limited at first.
-    let observer_join_handle = observer_manager.start().await;
+    let _observer_join_handle = observer_manager.start().await;
 
     let sstable_object_id_manager = Arc::new(SstableObjectIdManager::new(
         hummock_meta_client.clone(),
@@ -259,6 +259,8 @@ pub async fn compactor_serve(
         task_progress_manager: Default::default(),
         await_tree_reg: await_tree_reg.clone(),
     };
+
+    // TODO(shutdown): don't collect sub-tasks as there's no need to gracefully shutdown them.
     let mut sub_tasks = vec![
         MetaClient::start_heartbeat_loop(
             meta_client.clone(),
@@ -287,50 +289,42 @@ pub async fn compactor_serve(
 
     let compactor_srv = CompactorServiceImpl::default();
     let monitor_srv = MonitorServiceImpl::new(await_tree_reg);
-    let (shutdown_send, mut shutdown_recv) = tokio::sync::oneshot::channel();
-    let join_handle = tokio::spawn(async move {
-        tonic::transport::Server::builder()
-            .add_service(CompactorServiceServer::new(compactor_srv))
-            .add_service(MonitorServiceServer::new(monitor_srv))
-            .monitored_serve_with_shutdown(
-                listen_addr,
-                "grpc-compactor-node-service",
-                TcpConfig {
-                    tcp_nodelay: true,
-                    keepalive_duration: None,
-                },
-                async move {
-                    tokio::select! {
-                        _ = tokio::signal::ctrl_c() => {},
-                        _ = &mut shutdown_recv => {
-                            for (join_handle, shutdown_sender) in sub_tasks {
-                                if shutdown_sender.send(()).is_err() {
-                                    tracing::warn!("Failed to send shutdown");
-                                    continue;
-                                }
-                                if join_handle.await.is_err() {
-                                    tracing::warn!("Failed to join shutdown");
-                                }
-                            }
-                        },
-                    }
-                },
-            )
-            .await
-    });
+    let server = tonic::transport::Server::builder()
+        .add_service(CompactorServiceServer::new(compactor_srv))
+        .add_service(MonitorServiceServer::new(monitor_srv))
+        .monitored_serve_with_shutdown(
+            listen_addr,
+            "grpc-compactor-node-service",
+            TcpConfig {
+                tcp_nodelay: true,
+                keepalive_duration: None,
+            },
+            shutdown.clone().cancelled_owned(),
+        );
+    let _server_handle = tokio::spawn(server);
 
     // Boot metrics service.
     if config.server.metrics_level > MetricLevel::Disabled {
         MetricsManager::boot_metrics_service(opts.prometheus_listener_addr.clone());
     }
 
-    (join_handle, observer_join_handle, shutdown_send)
+    // All set, let the meta service know we're ready.
+    meta_client.activate(&advertise_addr).await.unwrap();
+
+    // Wait for the shutdown signal.
+    shutdown.cancelled().await;
+    // Run shutdown logic.
+    meta_client.try_unregister().await;
 }
 
+/// Fetches and runs compaction tasks under shared mode.
+///
+/// Returns when the `shutdown` token is triggered.
 pub async fn shared_compactor_serve(
     listen_addr: SocketAddr,
     opts: CompactorOpts,
-) -> (JoinHandle<()>, Sender<()>) {
+    shutdown: CancellationToken,
+) {
     let config = load_config(&opts.config_path, &opts);
     info!("Starting shared compactor node",);
     info!("> config: {:?}", config);
@@ -373,7 +367,6 @@ pub async fn shared_compactor_serve(
     // Run a background heap profiler
     heap_profiler.start();
 
-    let (shutdown_send, mut shutdown_recv) = tokio::sync::oneshot::channel();
     let compaction_executor = Arc::new(CompactionExecutor::new(
         opts.compaction_worker_threads_number,
     ));
@@ -387,44 +380,35 @@ pub async fn shared_compactor_serve(
         task_progress_manager: Default::default(),
         await_tree_reg,
     };
-    let join_handle = tokio::spawn(async move {
-        tonic::transport::Server::builder()
-            .add_service(CompactorServiceServer::new(compactor_srv))
-            .add_service(MonitorServiceServer::new(monitor_srv))
-            .monitored_serve_with_shutdown(
-                listen_addr,
-                "grpc-compactor-node-service",
-                TcpConfig {
-                    tcp_nodelay: true,
-                    keepalive_duration: None,
-                },
-                async move {
-                    let (join_handle, shutdown_sender) =
-                        risingwave_storage::hummock::compactor::start_shared_compactor(
-                            grpc_proxy_client,
-                            receiver,
-                            compactor_context,
-                        );
-                    tokio::select! {
-                        _ = tokio::signal::ctrl_c() => {},
-                        _ = &mut shutdown_recv => {
-                            if shutdown_sender.send(()).is_err() {
-                                tracing::warn!("Failed to send shutdown");
-                            }
-                            if join_handle.await.is_err() {
-                                tracing::warn!("Failed to join shutdown");
-                            }
-                        },
-                    }
-                },
-            )
-            .await
-    });
+
+    risingwave_storage::hummock::compactor::start_shared_compactor(
+        grpc_proxy_client,
+        receiver,
+        compactor_context,
+    );
+
+    let server = tonic::transport::Server::builder()
+        .add_service(CompactorServiceServer::new(compactor_srv))
+        .add_service(MonitorServiceServer::new(monitor_srv))
+        .monitored_serve_with_shutdown(
+            listen_addr,
+            "grpc-compactor-node-service",
+            TcpConfig {
+                tcp_nodelay: true,
+                keepalive_duration: None,
+            },
+            shutdown.clone().cancelled_owned(),
+        );
+
+    let _server_handle = tokio::spawn(server);
 
     // Boot metrics service.
     if config.server.metrics_level > MetricLevel::Disabled {
         MetricsManager::boot_metrics_service(opts.prometheus_listener_addr.clone());
     }
 
-    (join_handle, shutdown_send)
+    // Wait for the shutdown signal.
+    shutdown.cancelled().await;
+
+    // TODO(shutdown): shall we notify the proxy that we are shutting down?
 }
diff --git a/src/storage/compactor/src/telemetry.rs b/src/storage/compactor/src/telemetry.rs
index b01fdd1e258f3..815b485f9e898 100644
--- a/src/storage/compactor/src/telemetry.rs
+++ b/src/storage/compactor/src/telemetry.rs
@@ -14,14 +14,35 @@
 
 use prost::Message;
 use risingwave_common::telemetry::pb_compatible::TelemetryToProtobuf;
-use risingwave_common::telemetry::report::TelemetryReportCreator;
+use risingwave_common::telemetry::report::{report_event_common, TelemetryReportCreator};
 use risingwave_common::telemetry::{
     current_timestamp, SystemData, TelemetryNodeType, TelemetryReportBase, TelemetryResult,
 };
+use risingwave_pb::telemetry::{PbTelemetryDatabaseObject, PbTelemetryEventStage};
 use serde::{Deserialize, Serialize};
 
 const TELEMETRY_COMPACTOR_REPORT_TYPE: &str = "compactor";
 
+#[allow(dead_code)] // please remove when used
+pub(crate) fn report_event(
+    event_stage: PbTelemetryEventStage,
+    event_name: &str,
+    catalog_id: i64,
+    connector_name: Option<String>,
+    object: Option<PbTelemetryDatabaseObject>,
+    attributes: Option<jsonbb::Value>, // json object
+) {
+    report_event_common(
+        event_stage,
+        event_name,
+        catalog_id,
+        connector_name,
+        object,
+        attributes,
+        TELEMETRY_COMPACTOR_REPORT_TYPE.to_string(),
+    );
+}
+
 #[derive(Clone, Copy)]
 pub(crate) struct CompactorTelemetryCreator {}
 
diff --git a/src/storage/hummock_sdk/src/change_log.rs b/src/storage/hummock_sdk/src/change_log.rs
index 49224c92091f5..885880933ce11 100644
--- a/src/storage/hummock_sdk/src/change_log.rs
+++ b/src/storage/hummock_sdk/src/change_log.rs
@@ -15,13 +15,70 @@
 use std::collections::HashMap;
 
 use risingwave_common::catalog::TableId;
-use risingwave_pb::hummock::hummock_version_delta::ChangeLogDelta;
-use risingwave_pb::hummock::{EpochNewChangeLog, SstableInfo, TableChangeLog as PbTableChangeLog};
+use risingwave_pb::hummock::hummock_version_delta::PbChangeLogDelta;
+use risingwave_pb::hummock::{PbEpochNewChangeLog, PbTableChangeLog};
 use tracing::warn;
 
+use crate::sstable_info::SstableInfo;
+
 #[derive(Debug, Clone, PartialEq)]
 pub struct TableChangeLog(pub Vec<EpochNewChangeLog>);
 
+#[derive(Debug, Clone, PartialEq)]
+pub struct EpochNewChangeLog {
+    pub new_value: Vec<SstableInfo>,
+    pub old_value: Vec<SstableInfo>,
+    pub epochs: Vec<u64>,
+}
+
+impl From<&EpochNewChangeLog> for PbEpochNewChangeLog {
+    fn from(val: &EpochNewChangeLog) -> Self {
+        Self {
+            new_value: val.new_value.iter().map(|a| a.clone().into()).collect(),
+            old_value: val.old_value.iter().map(|a| a.clone().into()).collect(),
+            epochs: val.epochs.clone(),
+        }
+    }
+}
+
+impl From<&PbEpochNewChangeLog> for EpochNewChangeLog {
+    fn from(value: &PbEpochNewChangeLog) -> Self {
+        Self {
+            new_value: value.new_value.iter().map(|a| a.into()).collect(),
+            old_value: value.old_value.iter().map(|a| a.into()).collect(),
+            epochs: value.epochs.clone(),
+        }
+    }
+}
+
+impl From<EpochNewChangeLog> for PbEpochNewChangeLog {
+    fn from(val: EpochNewChangeLog) -> Self {
+        Self {
+            new_value: val
+                .new_value
+                .into_iter()
+                .map(|a| a.clone().into())
+                .collect(),
+            old_value: val
+                .old_value
+                .into_iter()
+                .map(|a| a.clone().into())
+                .collect(),
+            epochs: val.epochs.clone(),
+        }
+    }
+}
+
+impl From<PbEpochNewChangeLog> for EpochNewChangeLog {
+    fn from(value: PbEpochNewChangeLog) -> Self {
+        Self {
+            new_value: value.new_value.into_iter().map(|a| a.into()).collect(),
+            old_value: value.old_value.into_iter().map(|a| a.into()).collect(),
+            epochs: value.epochs.clone(),
+        }
+    }
+}
+
 impl TableChangeLog {
     pub fn filter_epoch(&self, (min_epoch, max_epoch): (u64, u64)) -> &[EpochNewChangeLog] {
         let start = self.0.partition_point(|epoch_change_log| {
@@ -56,12 +113,12 @@ impl TableChangeLog {
 impl TableChangeLog {
     pub fn to_protobuf(&self) -> PbTableChangeLog {
         PbTableChangeLog {
-            change_logs: self.0.clone(),
+            change_logs: self.0.iter().map(|a| a.into()).collect(),
         }
     }
 
     pub fn from_protobuf(val: &PbTableChangeLog) -> Self {
-        Self(val.change_logs.clone())
+        Self(val.change_logs.clone().iter().map(|a| a.into()).collect())
     }
 }
 
@@ -108,6 +165,48 @@ pub fn build_table_change_log_delta<'a>(
     table_change_log
 }
 
+#[derive(Debug, PartialEq, Clone)]
+pub struct ChangeLogDelta {
+    pub truncate_epoch: u64,
+    pub new_log: Option<EpochNewChangeLog>,
+}
+
+impl From<&ChangeLogDelta> for PbChangeLogDelta {
+    fn from(val: &ChangeLogDelta) -> Self {
+        Self {
+            truncate_epoch: val.truncate_epoch,
+            new_log: val.new_log.as_ref().map(|a| a.into()),
+        }
+    }
+}
+
+impl From<&PbChangeLogDelta> for ChangeLogDelta {
+    fn from(val: &PbChangeLogDelta) -> Self {
+        Self {
+            truncate_epoch: val.truncate_epoch,
+            new_log: val.new_log.as_ref().map(|a| a.into()),
+        }
+    }
+}
+
+impl From<ChangeLogDelta> for PbChangeLogDelta {
+    fn from(val: ChangeLogDelta) -> Self {
+        Self {
+            truncate_epoch: val.truncate_epoch,
+            new_log: val.new_log.map(|a| a.into()),
+        }
+    }
+}
+
+impl From<PbChangeLogDelta> for ChangeLogDelta {
+    fn from(val: PbChangeLogDelta) -> Self {
+        Self {
+            truncate_epoch: val.truncate_epoch,
+            new_log: val.new_log.map(|a| a.into()),
+        }
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use itertools::Itertools;
diff --git a/src/storage/hummock_sdk/src/compact.rs b/src/storage/hummock_sdk/src/compact.rs
index d6824ab646c43..ff12414477916 100644
--- a/src/storage/hummock_sdk/src/compact.rs
+++ b/src/storage/hummock_sdk/src/compact.rs
@@ -14,7 +14,10 @@
 
 use std::collections::HashSet;
 
-use risingwave_pb::hummock::{CompactTask, LevelType, SstableInfo};
+use risingwave_pb::hummock::LevelType;
+
+use crate::compact_task::CompactTask;
+use crate::sstable_info::SstableInfo;
 
 pub fn compact_task_output_to_string(compact_task: &CompactTask) -> String {
     use std::fmt::Write;
@@ -25,13 +28,13 @@ pub fn compact_task_output_to_string(compact_task: &CompactTask) -> String {
         "Compaction task id: {:?}, group-id: {:?}, type: {:?}, target level: {:?}, target sub level: {:?} watermark: {:?}, target_file_size: {:?}, splits: {:?}, status: {:?}",
         compact_task.task_id,
         compact_task.compaction_group_id,
-        compact_task.task_type(),
+        compact_task.task_type,
         compact_task.target_level,
         compact_task.target_sub_level_id,
         compact_task.watermark,
         compact_task.target_file_size,
         compact_task.splits.len(),
-        compact_task.task_status()
+        compact_task.task_status
     )
     .unwrap();
     s.push_str("Output: \n");
@@ -50,7 +53,7 @@ pub fn compact_task_to_string(compact_task: &CompactTask) -> String {
         "Compaction task id: {:?}, group-id: {:?}, type: {:?}, target level: {:?}, target sub level: {:?} watermark: {:?}, target_file_size: {:?}, splits: {:?}",
         compact_task.task_id,
         compact_task.compaction_group_id,
-        compact_task.task_type(),
+        compact_task.task_type,
         compact_task.target_level,
         compact_task.target_sub_level_id,
         compact_task.watermark,
@@ -81,7 +84,7 @@ pub fn compact_task_to_string(compact_task: &CompactTask) -> String {
                 if table.total_key_count != 0 {
                     format!(
                         "[id: {}, obj_id: {} {}KB stale_ratio {}]",
-                        table.get_sst_id(),
+                        table.sst_id,
                         table.object_id,
                         table.file_size / 1024,
                         (table.stale_key_count * 100 / table.total_key_count),
@@ -89,7 +92,7 @@ pub fn compact_task_to_string(compact_task: &CompactTask) -> String {
                 } else {
                     format!(
                         "[id: {}, obj_id: {} {}KB]",
-                        table.get_sst_id(),
+                        table.sst_id,
                         table.object_id,
                         table.file_size / 1024,
                     )
@@ -118,16 +121,16 @@ pub fn compact_task_to_string(compact_task: &CompactTask) -> String {
 pub fn append_sstable_info_to_string(s: &mut String, sstable_info: &SstableInfo) {
     use std::fmt::Write;
 
-    let key_range = sstable_info.key_range.as_ref().unwrap();
+    let key_range = &sstable_info.key_range;
     let left_str = if key_range.left.is_empty() {
         "-inf".to_string()
     } else {
-        hex::encode(key_range.left.as_slice())
+        hex::encode(&key_range.left)
     };
     let right_str = if key_range.right.is_empty() {
         "+inf".to_string()
     } else {
-        hex::encode(key_range.right.as_slice())
+        hex::encode(&key_range.right)
     };
 
     let stale_ratio = (sstable_info.stale_key_count * 100)
@@ -136,8 +139,8 @@ pub fn append_sstable_info_to_string(s: &mut String, sstable_info: &SstableInfo)
     writeln!(
         s,
         "SstableInfo: object id={}, SST id={}, KeyRange=[{:?},{:?}], table_ids: {:?}, size={}KB, stale_ratio={}%, bloom_filter_kind {:?}",
-        sstable_info.get_object_id(),
-        sstable_info.get_sst_id(),
+        sstable_info.object_id,
+        sstable_info.sst_id,
         left_str,
         right_str,
         sstable_info.table_ids,
@@ -158,8 +161,8 @@ pub fn statistics_compact_task(task: &CompactTask) -> CompactTaskStatistics {
         total_file_count += level.table_infos.len() as u64;
 
         level.table_infos.iter().for_each(|sst| {
-            total_file_size += sst.get_file_size();
-            total_uncompressed_file_size += sst.get_uncompressed_file_size();
+            total_file_size += sst.file_size;
+            total_uncompressed_file_size += sst.uncompressed_file_size;
             total_key_count += sst.total_key_count;
         });
     }
@@ -200,7 +203,7 @@ pub fn estimate_memory_for_compact_task(
 
     // input
     for level in &task.input_ssts {
-        if level.level_type() == LevelType::Nonoverlapping {
+        if level.level_type == LevelType::Nonoverlapping {
             let mut cur_level_max_sst_meta_size = 0;
             for sst in &level.table_infos {
                 let meta_size = sst.file_size - sst.meta_offset;
diff --git a/src/storage/hummock_sdk/src/compact_task.rs b/src/storage/hummock_sdk/src/compact_task.rs
new file mode 100644
index 0000000000000..61f96e1ee3b52
--- /dev/null
+++ b/src/storage/hummock_sdk/src/compact_task.rs
@@ -0,0 +1,412 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::{BTreeMap, HashMap};
+use std::mem::size_of;
+
+use itertools::Itertools;
+use risingwave_pb::hummock::compact_task::{PbTaskStatus, PbTaskType, TaskStatus};
+use risingwave_pb::hummock::subscribe_compaction_event_request::PbReportTask;
+use risingwave_pb::hummock::{
+    PbCompactTask, PbKeyRange, PbTableOption, PbTableSchema, PbTableStats, PbValidationTask,
+};
+
+use crate::key_range::KeyRange;
+use crate::level::InputLevel;
+use crate::sstable_info::SstableInfo;
+use crate::table_watermark::TableWatermarks;
+
+#[derive(Clone, PartialEq, Default, Debug)]
+pub struct CompactTask {
+    /// SSTs to be compacted, which will be removed from LSM after compaction
+    pub input_ssts: Vec<InputLevel>,
+    /// In ideal case, the compaction will generate `splits.len()` tables which have key range
+    /// corresponding to that in `splits`, respectively
+    pub splits: Vec<KeyRange>,
+    /// low watermark in 'ts-aware compaction'
+    pub watermark: u64,
+    /// compaction output, which will be added to `target_level` of LSM after compaction
+    pub sorted_output_ssts: Vec<SstableInfo>,
+    /// task id assigned by hummock storage service
+    pub task_id: u64,
+    /// compaction output will be added to `target_level` of LSM after compaction
+    pub target_level: u32,
+    pub gc_delete_keys: bool,
+    /// Lbase in LSM
+    pub base_level: u32,
+    pub task_status: PbTaskStatus,
+    /// compaction group the task belongs to
+    pub compaction_group_id: u64,
+    /// `existing_table_ids` for compaction drop key
+    pub existing_table_ids: Vec<u32>,
+    pub compression_algorithm: u32,
+    pub target_file_size: u64,
+    pub compaction_filter_mask: u32,
+    pub table_options: BTreeMap<u32, PbTableOption>,
+    pub current_epoch_time: u64,
+    pub target_sub_level_id: u64,
+    /// Identifies whether the task is `space_reclaim`, if the `compact_task_type` increases, it will be refactored to enum
+    pub task_type: PbTaskType,
+    /// Deprecated. use `table_vnode_partition` instead;
+    pub split_by_state_table: bool,
+    /// Compaction needs to cut the state table every time 1/weight of vnodes in the table have been processed.
+    /// Deprecated. use `table_vnode_partition` instead;
+    pub split_weight_by_vnode: u32,
+    pub table_vnode_partition: BTreeMap<u32, u32>,
+    /// The table watermark of any table id. In compaction we only use the table watermarks on safe epoch,
+    /// so we only need to include the table watermarks on safe epoch to reduce the size of metadata.
+    pub table_watermarks: BTreeMap<u32, TableWatermarks>,
+
+    pub table_schemas: BTreeMap<u32, PbTableSchema>,
+
+    pub max_sub_compaction: u32,
+}
+
+impl CompactTask {
+    pub fn estimated_encode_len(&self) -> usize {
+        self.input_ssts
+            .iter()
+            .map(|input_level| input_level.estimated_encode_len())
+            .sum::<usize>()
+            + self
+                .splits
+                .iter()
+                .map(|split| split.left.len() + split.right.len() + size_of::<bool>())
+                .sum::<usize>()
+            + size_of::<u64>()
+            + self
+                .sorted_output_ssts
+                .iter()
+                .map(|sst| sst.estimated_encode_len())
+                .sum::<usize>()
+            + size_of::<u64>()
+            + size_of::<u32>()
+            + size_of::<bool>()
+            + size_of::<u32>()
+            + size_of::<i32>()
+            + size_of::<u64>()
+            + self.existing_table_ids.len() * size_of::<u32>()
+            + size_of::<u32>()
+            + size_of::<u64>()
+            + size_of::<u32>()
+            + self.table_options.len() * size_of::<u64>()
+            + size_of::<u64>()
+            + size_of::<u64>()
+            + size_of::<i32>()
+            + size_of::<bool>()
+            + size_of::<u32>()
+            + self.table_vnode_partition.len() * size_of::<u64>()
+            + self
+                .table_watermarks
+                .values()
+                .map(|table_watermark| size_of::<u32>() + table_watermark.estimated_encode_len())
+                .sum::<usize>()
+    }
+}
+
+impl From<PbCompactTask> for CompactTask {
+    #[expect(deprecated)]
+    fn from(pb_compact_task: PbCompactTask) -> Self {
+        Self {
+            input_ssts: pb_compact_task
+                .input_ssts
+                .into_iter()
+                .map(InputLevel::from)
+                .collect_vec(),
+            splits: pb_compact_task
+                .splits
+                .into_iter()
+                .map(|pb_keyrange| KeyRange {
+                    left: pb_keyrange.left.into(),
+                    right: pb_keyrange.right.into(),
+                    right_exclusive: pb_keyrange.right_exclusive,
+                })
+                .collect_vec(),
+            watermark: pb_compact_task.watermark,
+            sorted_output_ssts: pb_compact_task
+                .sorted_output_ssts
+                .into_iter()
+                .map(SstableInfo::from)
+                .collect_vec(),
+            task_id: pb_compact_task.task_id,
+            target_level: pb_compact_task.target_level,
+            gc_delete_keys: pb_compact_task.gc_delete_keys,
+            base_level: pb_compact_task.base_level,
+            task_status: TaskStatus::try_from(pb_compact_task.task_status).unwrap(),
+            compaction_group_id: pb_compact_task.compaction_group_id,
+            existing_table_ids: pb_compact_task.existing_table_ids.clone(),
+            compression_algorithm: pb_compact_task.compression_algorithm,
+            target_file_size: pb_compact_task.target_file_size,
+            compaction_filter_mask: pb_compact_task.compaction_filter_mask,
+            table_options: pb_compact_task.table_options.clone(),
+            current_epoch_time: pb_compact_task.current_epoch_time,
+            target_sub_level_id: pb_compact_task.target_sub_level_id,
+            task_type: PbTaskType::try_from(pb_compact_task.task_type).unwrap(),
+            split_by_state_table: pb_compact_task.split_by_state_table,
+            split_weight_by_vnode: pb_compact_task.split_weight_by_vnode,
+            table_vnode_partition: pb_compact_task.table_vnode_partition.clone(),
+            table_watermarks: pb_compact_task
+                .table_watermarks
+                .into_iter()
+                .map(|(table_id, pb_table_watermark)| {
+                    (table_id, TableWatermarks::from(pb_table_watermark))
+                })
+                .collect(),
+            table_schemas: pb_compact_task.table_schemas,
+            max_sub_compaction: pb_compact_task.max_sub_compaction,
+        }
+    }
+}
+
+impl From<&PbCompactTask> for CompactTask {
+    #[expect(deprecated)]
+    fn from(pb_compact_task: &PbCompactTask) -> Self {
+        Self {
+            input_ssts: pb_compact_task
+                .input_ssts
+                .iter()
+                .map(InputLevel::from)
+                .collect_vec(),
+            splits: pb_compact_task
+                .splits
+                .iter()
+                .map(|pb_keyrange| KeyRange {
+                    left: pb_keyrange.left.clone().into(),
+                    right: pb_keyrange.right.clone().into(),
+                    right_exclusive: pb_keyrange.right_exclusive,
+                })
+                .collect_vec(),
+            watermark: pb_compact_task.watermark,
+            sorted_output_ssts: pb_compact_task
+                .sorted_output_ssts
+                .iter()
+                .map(SstableInfo::from)
+                .collect_vec(),
+            task_id: pb_compact_task.task_id,
+            target_level: pb_compact_task.target_level,
+            gc_delete_keys: pb_compact_task.gc_delete_keys,
+            base_level: pb_compact_task.base_level,
+            task_status: TaskStatus::try_from(pb_compact_task.task_status).unwrap(),
+            compaction_group_id: pb_compact_task.compaction_group_id,
+            existing_table_ids: pb_compact_task.existing_table_ids.clone(),
+            compression_algorithm: pb_compact_task.compression_algorithm,
+            target_file_size: pb_compact_task.target_file_size,
+            compaction_filter_mask: pb_compact_task.compaction_filter_mask,
+            table_options: pb_compact_task.table_options.clone(),
+            current_epoch_time: pb_compact_task.current_epoch_time,
+            target_sub_level_id: pb_compact_task.target_sub_level_id,
+            task_type: PbTaskType::try_from(pb_compact_task.task_type).unwrap(),
+            split_by_state_table: pb_compact_task.split_by_state_table,
+            split_weight_by_vnode: pb_compact_task.split_weight_by_vnode,
+            table_vnode_partition: pb_compact_task.table_vnode_partition.clone(),
+            table_watermarks: pb_compact_task
+                .table_watermarks
+                .iter()
+                .map(|(table_id, pb_table_watermark)| {
+                    (*table_id, TableWatermarks::from(pb_table_watermark))
+                })
+                .collect(),
+            table_schemas: pb_compact_task.table_schemas.clone(),
+            max_sub_compaction: pb_compact_task.max_sub_compaction,
+        }
+    }
+}
+
+impl From<CompactTask> for PbCompactTask {
+    #[expect(deprecated)]
+    fn from(compact_task: CompactTask) -> Self {
+        Self {
+            input_ssts: compact_task
+                .input_ssts
+                .into_iter()
+                .map(|input_level| input_level.into())
+                .collect_vec(),
+            splits: compact_task
+                .splits
+                .into_iter()
+                .map(|keyrange| PbKeyRange {
+                    left: keyrange.left.into(),
+                    right: keyrange.right.into(),
+                    right_exclusive: keyrange.right_exclusive,
+                })
+                .collect_vec(),
+            watermark: compact_task.watermark,
+            sorted_output_ssts: compact_task
+                .sorted_output_ssts
+                .into_iter()
+                .map(|sst| sst.into())
+                .collect_vec(),
+            task_id: compact_task.task_id,
+            target_level: compact_task.target_level,
+            gc_delete_keys: compact_task.gc_delete_keys,
+            base_level: compact_task.base_level,
+            task_status: compact_task.task_status.into(),
+            compaction_group_id: compact_task.compaction_group_id,
+            existing_table_ids: compact_task.existing_table_ids.clone(),
+            compression_algorithm: compact_task.compression_algorithm,
+            target_file_size: compact_task.target_file_size,
+            compaction_filter_mask: compact_task.compaction_filter_mask,
+            table_options: compact_task.table_options.clone(),
+            current_epoch_time: compact_task.current_epoch_time,
+            target_sub_level_id: compact_task.target_sub_level_id,
+            task_type: compact_task.task_type.into(),
+            split_weight_by_vnode: compact_task.split_weight_by_vnode,
+            table_vnode_partition: compact_task.table_vnode_partition.clone(),
+            table_watermarks: compact_task
+                .table_watermarks
+                .into_iter()
+                .map(|(table_id, table_watermark)| (table_id, table_watermark.into()))
+                .collect(),
+            split_by_state_table: compact_task.split_by_state_table,
+            table_schemas: compact_task.table_schemas.clone(),
+            max_sub_compaction: compact_task.max_sub_compaction,
+        }
+    }
+}
+
+impl From<&CompactTask> for PbCompactTask {
+    #[expect(deprecated)]
+    fn from(compact_task: &CompactTask) -> Self {
+        Self {
+            input_ssts: compact_task
+                .input_ssts
+                .iter()
+                .map(|input_level| input_level.into())
+                .collect_vec(),
+            splits: compact_task
+                .splits
+                .iter()
+                .map(|keyrange| PbKeyRange {
+                    left: keyrange.left.to_vec(),
+                    right: keyrange.right.to_vec(),
+                    right_exclusive: keyrange.right_exclusive,
+                })
+                .collect_vec(),
+            watermark: compact_task.watermark,
+            sorted_output_ssts: compact_task
+                .sorted_output_ssts
+                .iter()
+                .map(|sst| sst.into())
+                .collect_vec(),
+            task_id: compact_task.task_id,
+            target_level: compact_task.target_level,
+            gc_delete_keys: compact_task.gc_delete_keys,
+            base_level: compact_task.base_level,
+            task_status: compact_task.task_status.into(),
+            compaction_group_id: compact_task.compaction_group_id,
+            existing_table_ids: compact_task.existing_table_ids.clone(),
+            compression_algorithm: compact_task.compression_algorithm,
+            target_file_size: compact_task.target_file_size,
+            compaction_filter_mask: compact_task.compaction_filter_mask,
+            table_options: compact_task.table_options.clone(),
+            current_epoch_time: compact_task.current_epoch_time,
+            target_sub_level_id: compact_task.target_sub_level_id,
+            task_type: compact_task.task_type.into(),
+            split_weight_by_vnode: compact_task.split_weight_by_vnode,
+            table_vnode_partition: compact_task.table_vnode_partition.clone(),
+            table_watermarks: compact_task
+                .table_watermarks
+                .iter()
+                .map(|(table_id, table_watermark)| (*table_id, table_watermark.into()))
+                .collect(),
+            split_by_state_table: compact_task.split_by_state_table,
+            table_schemas: compact_task.table_schemas.clone(),
+            max_sub_compaction: compact_task.max_sub_compaction,
+        }
+    }
+}
+
+#[derive(Clone, PartialEq, Default)]
+pub struct ValidationTask {
+    pub sst_infos: Vec<SstableInfo>,
+    pub sst_id_to_worker_id: HashMap<u64, u32>,
+    pub epoch: u64,
+}
+
+impl From<PbValidationTask> for ValidationTask {
+    fn from(pb_validation_task: PbValidationTask) -> Self {
+        Self {
+            sst_infos: pb_validation_task
+                .sst_infos
+                .into_iter()
+                .map(SstableInfo::from)
+                .collect_vec(),
+            sst_id_to_worker_id: pb_validation_task.sst_id_to_worker_id.clone(),
+            epoch: pb_validation_task.epoch,
+        }
+    }
+}
+
+impl From<ValidationTask> for PbValidationTask {
+    fn from(validation_task: ValidationTask) -> Self {
+        Self {
+            sst_infos: validation_task
+                .sst_infos
+                .into_iter()
+                .map(|sst| sst.into())
+                .collect_vec(),
+            sst_id_to_worker_id: validation_task.sst_id_to_worker_id.clone(),
+            epoch: validation_task.epoch,
+        }
+    }
+}
+
+impl ValidationTask {
+    pub fn estimated_encode_len(&self) -> usize {
+        self.sst_infos
+            .iter()
+            .map(|sst| sst.estimated_encode_len())
+            .sum::<usize>()
+            + self.sst_id_to_worker_id.len() * (size_of::<u64>() + size_of::<u32>())
+            + size_of::<u64>()
+    }
+}
+
+#[derive(Clone, PartialEq, Default, Debug)]
+pub struct ReportTask {
+    pub table_stats_change: HashMap<u32, PbTableStats>,
+    pub task_id: u64,
+    pub task_status: TaskStatus,
+    pub sorted_output_ssts: Vec<SstableInfo>,
+}
+
+impl From<PbReportTask> for ReportTask {
+    fn from(value: PbReportTask) -> Self {
+        Self {
+            table_stats_change: value.table_stats_change.clone(),
+            task_id: value.task_id,
+            task_status: PbTaskStatus::try_from(value.task_status).unwrap(),
+            sorted_output_ssts: value
+                .sorted_output_ssts
+                .into_iter()
+                .map(SstableInfo::from)
+                .collect_vec(),
+        }
+    }
+}
+
+impl From<ReportTask> for PbReportTask {
+    fn from(value: ReportTask) -> Self {
+        Self {
+            table_stats_change: value.table_stats_change.clone(),
+            task_id: value.task_id,
+            task_status: value.task_status.into(),
+            sorted_output_ssts: value
+                .sorted_output_ssts
+                .into_iter()
+                .map(|sst| sst.into())
+                .collect_vec(),
+        }
+    }
+}
diff --git a/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs b/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs
index 1aba344e166a0..0fe019d9afcd3 100644
--- a/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs
+++ b/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs
@@ -17,16 +17,13 @@ use std::collections::hash_map::Entry;
 use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
 use std::sync::Arc;
 
+use bytes::Bytes;
 use itertools::Itertools;
 use risingwave_common::catalog::TableId;
-use risingwave_pb::hummock::group_delta::DeltaType;
-use risingwave_pb::hummock::hummock_version::Levels;
-use risingwave_pb::hummock::hummock_version_delta::GroupDeltas;
-use risingwave_pb::hummock::table_watermarks::PbEpochNewWatermarks;
+use risingwave_common::hash::VnodeBitmapExt;
 use risingwave_pb::hummock::{
     CompactionConfig, CompatibilityVersion, GroupConstruct, GroupDestroy, GroupMetaChange,
-    GroupTableChange, Level, LevelType, OverlappingLevel, PbLevelType, PbTableWatermarks,
-    SstableInfo,
+    GroupTableChange, PbLevelType,
 };
 use tracing::warn;
 
@@ -34,9 +31,13 @@ use super::StateTableId;
 use crate::change_log::TableChangeLog;
 use crate::compaction_group::StaticCompactionGroupId;
 use crate::key_range::KeyRangeCommon;
-use crate::prost_key_range::KeyRangeExt;
-use crate::table_watermark::{TableWatermarks, VnodeWatermark};
-use crate::version::{HummockVersion, HummockVersionDelta};
+use crate::level::{Level, Levels, OverlappingLevel};
+use crate::sstable_info::SstableInfo;
+use crate::table_watermark::{ReadTableWatermark, TableWatermarks};
+use crate::version::{
+    GroupDelta, GroupDeltas, HummockVersion, HummockVersionDelta, HummockVersionStateTableInfo,
+    IntraLevelDelta,
+};
 use crate::{can_concat, CompactionGroupId, HummockSstableId, HummockSstableObjectId};
 
 pub struct GroupDeltasSummary {
@@ -65,8 +66,8 @@ pub fn summarize_group_deltas(group_deltas: &GroupDeltas) -> GroupDeltasSummary
     let mut new_vnode_partition_count = 0;
 
     for group_delta in &group_deltas.group_deltas {
-        match group_delta.get_delta_type().unwrap() {
-            DeltaType::IntraLevel(intra_level) => {
+        match group_delta {
+            GroupDelta::IntraLevel(intra_level) => {
                 if !intra_level.removed_table_ids.is_empty() {
                     delete_sst_levels.push(intra_level.level_idx);
                     delete_sst_ids_set.extend(intra_level.removed_table_ids.iter().clone());
@@ -78,18 +79,18 @@ pub fn summarize_group_deltas(group_deltas: &GroupDeltas) -> GroupDeltasSummary
                 }
                 new_vnode_partition_count = intra_level.vnode_partition_count;
             }
-            DeltaType::GroupConstruct(construct_delta) => {
+            GroupDelta::GroupConstruct(construct_delta) => {
                 assert!(group_construct.is_none());
                 group_construct = Some(construct_delta.clone());
             }
-            DeltaType::GroupDestroy(destroy_delta) => {
+            GroupDelta::GroupDestroy(destroy_delta) => {
                 assert!(group_destroy.is_none());
                 group_destroy = Some(destroy_delta.clone());
             }
-            DeltaType::GroupMetaChange(meta_delta) => {
+            GroupDelta::GroupMetaChange(meta_delta) => {
                 group_meta_changes.push(meta_delta.clone());
             }
-            DeltaType::GroupTableChange(meta_delta) => {
+            GroupDelta::GroupTableChange(meta_delta) => {
                 group_table_change = Some(meta_delta.clone());
             }
         }
@@ -146,36 +147,59 @@ impl HummockVersion {
     }
 
     pub fn get_combined_levels(&self) -> impl Iterator<Item = &'_ Level> + '_ {
-        self.levels.values().flat_map(|level| {
-            level
-                .l0
-                .as_ref()
-                .unwrap()
-                .sub_levels
-                .iter()
-                .rev()
-                .chain(level.levels.iter())
-        })
+        self.levels
+            .values()
+            .flat_map(|level| level.l0.sub_levels.iter().rev().chain(level.levels.iter()))
     }
 
     pub fn get_object_ids(&self) -> HashSet<HummockSstableObjectId> {
+        self.get_sst_infos().map(|s| s.object_id).collect()
+    }
+
+    pub fn get_sst_ids(&self) -> HashSet<HummockSstableObjectId> {
+        self.get_sst_infos().map(|s| s.sst_id).collect()
+    }
+
+    pub fn get_sst_infos(&self) -> impl Iterator<Item = &SstableInfo> {
         self.get_combined_levels()
-            .flat_map(|level| {
-                level
-                    .table_infos
-                    .iter()
-                    .map(|table_info| table_info.get_object_id())
+            .flat_map(|level| level.table_infos.iter())
+            .chain(self.table_change_log.values().flat_map(|change_log| {
+                change_log.0.iter().flat_map(|epoch_change_log| {
+                    epoch_change_log
+                        .old_value
+                        .iter()
+                        .chain(epoch_change_log.new_value.iter())
+                })
+            }))
+    }
+
+    /// `get_sst_infos_from_groups` doesn't guarantee that all returned sst info belongs to `select_group`.
+    /// i.e. `select_group` is just a hint.
+    /// We separate `get_sst_infos_from_groups` and `get_sst_infos` because `get_sst_infos_from_groups` may be further customized in the future.
+    pub fn get_sst_infos_from_groups<'a>(
+        &'a self,
+        select_group: &'a HashSet<CompactionGroupId>,
+    ) -> impl Iterator<Item = &SstableInfo> + 'a {
+        self.levels
+            .iter()
+            .filter_map(|(cg_id, level)| {
+                if select_group.contains(cg_id) {
+                    Some(level)
+                } else {
+                    None
+                }
             })
+            .flat_map(|level| level.l0.sub_levels.iter().rev().chain(level.levels.iter()))
+            .flat_map(|level| level.table_infos.iter())
             .chain(self.table_change_log.values().flat_map(|change_log| {
+                // TODO: optimization: strip table change log
                 change_log.0.iter().flat_map(|epoch_change_log| {
                     epoch_change_log
                         .old_value
                         .iter()
-                        .map(|sst| sst.object_id)
-                        .chain(epoch_change_log.new_value.iter().map(|sst| sst.object_id))
+                        .chain(epoch_change_log.new_value.iter())
                 })
             }))
-            .collect()
     }
 
     pub fn level_iter<F: FnMut(&Level) -> bool>(
@@ -184,7 +208,7 @@ impl HummockVersion {
         mut f: F,
     ) {
         if let Some(levels) = self.levels.get(&compaction_group_id) {
-            for sub_level in &levels.l0.as_ref().unwrap().sub_levels {
+            for sub_level in &levels.l0.sub_levels {
                 if !f(sub_level) {
                     return;
                 }
@@ -208,57 +232,94 @@ impl HummockVersion {
     pub fn safe_epoch_table_watermarks(
         &self,
         existing_table_ids: &[u32],
-    ) -> BTreeMap<u32, PbTableWatermarks> {
-        fn extract_single_table_watermark(
-            table_watermarks: &TableWatermarks,
-            safe_epoch: u64,
-        ) -> Option<PbTableWatermarks> {
-            if let Some((first_epoch, first_epoch_watermark)) = table_watermarks.watermarks.first()
-            {
-                assert!(
-                    *first_epoch >= safe_epoch,
-                    "smallest epoch {} in table watermark should be at least safe epoch {}",
-                    first_epoch,
-                    safe_epoch
-                );
-                if *first_epoch == safe_epoch {
-                    Some(PbTableWatermarks {
-                        epoch_watermarks: vec![PbEpochNewWatermarks {
-                            watermarks: first_epoch_watermark
-                                .iter()
-                                .map(VnodeWatermark::to_protobuf)
-                                .collect(),
-                            epoch: *first_epoch,
-                        }],
-                        is_ascending: table_watermarks.direction.is_ascending(),
-                    })
-                } else {
-                    None
-                }
+    ) -> BTreeMap<u32, TableWatermarks> {
+        safe_epoch_table_watermarks_impl(
+            &self.table_watermarks,
+            &self.state_table_info,
+            existing_table_ids,
+        )
+    }
+}
+
+pub fn safe_epoch_table_watermarks_impl(
+    table_watermarks: &HashMap<TableId, Arc<TableWatermarks>>,
+    state_table_info: &HummockVersionStateTableInfo,
+    existing_table_ids: &[u32],
+) -> BTreeMap<u32, TableWatermarks> {
+    fn extract_single_table_watermark(
+        table_watermarks: &TableWatermarks,
+        safe_epoch: u64,
+    ) -> Option<TableWatermarks> {
+        if let Some((first_epoch, first_epoch_watermark)) = table_watermarks.watermarks.first() {
+            assert!(
+                *first_epoch >= safe_epoch,
+                "smallest epoch {} in table watermark should be at least safe epoch {}",
+                first_epoch,
+                safe_epoch
+            );
+            if *first_epoch == safe_epoch {
+                Some(TableWatermarks {
+                    watermarks: vec![(*first_epoch, first_epoch_watermark.clone())],
+                    direction: table_watermarks.direction,
+                })
             } else {
                 None
             }
+        } else {
+            None
         }
-        self.table_watermarks
-            .iter()
-            .filter_map(|(table_id, table_watermarks)| {
-                let u32_table_id = table_id.table_id();
-                if !existing_table_ids.contains(&u32_table_id) {
-                    None
-                } else {
-                    extract_single_table_watermark(
-                        table_watermarks,
-                        self.state_table_info
-                            .info()
-                            .get(table_id)
-                            .expect("table should exist")
-                            .safe_epoch,
-                    )
-                    .map(|table_watermarks| (table_id.table_id, table_watermarks))
-                }
-            })
-            .collect()
     }
+    table_watermarks
+        .iter()
+        .filter_map(|(table_id, table_watermarks)| {
+            let u32_table_id = table_id.table_id();
+            if !existing_table_ids.contains(&u32_table_id) {
+                None
+            } else {
+                extract_single_table_watermark(
+                    table_watermarks,
+                    state_table_info
+                        .info()
+                        .get(table_id)
+                        .expect("table should exist")
+                        .safe_epoch,
+                )
+                .map(|table_watermarks| (table_id.table_id, table_watermarks))
+            }
+        })
+        .collect()
+}
+
+pub fn safe_epoch_read_table_watermarks_impl(
+    safe_epoch_watermarks: &BTreeMap<u32, TableWatermarks>,
+) -> BTreeMap<TableId, ReadTableWatermark> {
+    safe_epoch_watermarks
+        .iter()
+        .map(|(table_id, watermarks)| {
+            assert_eq!(watermarks.watermarks.len(), 1);
+            let vnode_watermarks = &watermarks.watermarks.first().expect("should exist").1;
+            let mut vnode_watermark_map = BTreeMap::new();
+            for vnode_watermark in vnode_watermarks.iter() {
+                let watermark = Bytes::copy_from_slice(vnode_watermark.watermark());
+                for vnode in vnode_watermark.vnode_bitmap().iter_vnodes() {
+                    assert!(
+                        vnode_watermark_map
+                            .insert(vnode, watermark.clone())
+                            .is_none(),
+                        "duplicate table watermark on vnode {}",
+                        vnode.to_index()
+                    );
+                }
+            }
+            (
+                TableId::from(*table_id),
+                ReadTableWatermark {
+                    direction: watermarks.direction,
+                    vnode_watermarks: vnode_watermark_map,
+                },
+            )
+        })
+        .collect()
 }
 
 impl HummockVersion {
@@ -272,13 +333,13 @@ impl HummockVersion {
             .map_or(0, |parent_levels| {
                 parent_levels
                     .l0
+                    .sub_levels
                     .iter()
-                    .flat_map(|l0| l0.get_sub_levels())
-                    .chain(parent_levels.get_levels().iter())
-                    .flat_map(|level| level.get_table_infos())
+                    .chain(parent_levels.levels.iter())
+                    .flat_map(|level| &level.table_infos)
                     .map(|sst_info| {
                         // `sst_info.table_ids` will never be empty.
-                        for table_id in sst_info.get_table_ids() {
+                        for table_id in &sst_info.table_ids {
                             if member_table_ids.contains(table_id) {
                                 return 2;
                             }
@@ -295,21 +356,36 @@ impl HummockVersion {
         group_id: CompactionGroupId,
         member_table_ids: HashSet<StateTableId>,
         new_sst_start_id: u64,
-        allow_trivial_split: bool,
     ) {
         let mut new_sst_id = new_sst_start_id;
-        if parent_group_id == StaticCompactionGroupId::NewCompactionGroup as CompactionGroupId
-            || !self.levels.contains_key(&parent_group_id)
-        {
+        if !self.levels.contains_key(&parent_group_id) {
+            warn!(parent_group_id, "non-existing parent group id to init from");
+            return;
+        }
+        if parent_group_id == StaticCompactionGroupId::NewCompactionGroup as CompactionGroupId {
+            if new_sst_start_id != 0 {
+                if cfg!(debug_assertions) {
+                    panic!(
+                        "non-zero sst start id {} for NewCompactionGroup",
+                        new_sst_start_id
+                    );
+                } else {
+                    warn!(
+                        new_sst_start_id,
+                        "non-zero sst start id for NewCompactionGroup"
+                    );
+                }
+            }
             return;
         }
         let [parent_levels, cur_levels] = self
             .levels
             .get_many_mut([&parent_group_id, &group_id])
             .unwrap();
-        if let Some(ref mut l0) = parent_levels.l0 {
+        let l0 = &mut parent_levels.l0;
+        {
             for sub_level in &mut l0.sub_levels {
-                let target_l0 = cur_levels.l0.as_mut().unwrap();
+                let target_l0 = &mut cur_levels.l0;
                 // When `insert_hint` is `Ok(idx)`, it means that the sub level `idx` in `target_l0`
                 // will extend these SSTs. When `insert_hint` is `Err(idx)`, it
                 // means that we will add a new sub level `idx` into `target_l0`.
@@ -329,12 +405,8 @@ impl HummockVersion {
                 }
                 // Remove SST from sub level may result in empty sub level. It will be purged
                 // whenever another compaction task is finished.
-                let insert_table_infos = split_sst_info_for_level(
-                    &member_table_ids,
-                    allow_trivial_split,
-                    sub_level,
-                    &mut new_sst_id,
-                );
+                let insert_table_infos =
+                    split_sst_info_for_level(&member_table_ids, sub_level, &mut new_sst_id);
                 sub_level
                     .table_infos
                     .extract_if(|sst_info| sst_info.table_ids.is_empty())
@@ -355,7 +427,7 @@ impl HummockVersion {
                         insert_new_sub_level(
                             target_l0,
                             sub_level.sub_level_id,
-                            sub_level.level_type(),
+                            sub_level.level_type,
                             insert_table_infos,
                             Some(idx),
                         );
@@ -364,12 +436,8 @@ impl HummockVersion {
             }
         }
         for (idx, level) in parent_levels.levels.iter_mut().enumerate() {
-            let insert_table_infos = split_sst_info_for_level(
-                &member_table_ids,
-                allow_trivial_split,
-                level,
-                &mut new_sst_id,
-            );
+            let insert_table_infos =
+                split_sst_info_for_level(&member_table_ids, level, &mut new_sst_id);
             cur_levels.levels[idx].total_file_size += insert_table_infos
                 .iter()
                 .map(|sst| sst.file_size)
@@ -381,11 +449,9 @@ impl HummockVersion {
             cur_levels.levels[idx]
                 .table_infos
                 .extend(insert_table_infos);
-            cur_levels.levels[idx].table_infos.sort_by(|sst1, sst2| {
-                let a = sst1.key_range.as_ref().unwrap();
-                let b = sst2.key_range.as_ref().unwrap();
-                a.compare(b)
-            });
+            cur_levels.levels[idx]
+                .table_infos
+                .sort_by(|sst1, sst2| sst1.key_range.cmp(&sst2.key_range));
             assert!(can_concat(&cur_levels.levels[idx].table_infos));
             level
                 .table_infos
@@ -400,6 +466,12 @@ impl HummockVersion {
     pub fn build_sst_delta_infos(&self, version_delta: &HummockVersionDelta) -> Vec<SstDeltaInfo> {
         let mut infos = vec![];
 
+        // Skip trivial move delta for refiller
+        // The trivial move task only changes the position of the sst in the lsm, it does not modify the object information corresponding to the sst, and does not need to re-execute the refill.
+        if version_delta.trivial_move {
+            return infos;
+        }
+
         for (group_id, group_deltas) in &version_delta.group_deltas {
             let mut info = SstDeltaInfo::default();
 
@@ -410,7 +482,7 @@ impl HummockVersion {
             if !group_deltas
                 .group_deltas
                 .iter()
-                .all(|delta| matches!(delta.get_delta_type().unwrap(), DeltaType::IntraLevel(..)))
+                .all(|delta| matches!(delta, GroupDelta::IntraLevel(_)))
             {
                 continue;
             }
@@ -419,18 +491,21 @@ impl HummockVersion {
             // current `hummock::manager::gen_version_delta` implementation. Better refactor the
             // struct to reduce conventions.
             for group_delta in &group_deltas.group_deltas {
-                if let DeltaType::IntraLevel(delta) = group_delta.get_delta_type().unwrap() {
-                    if !delta.inserted_table_infos.is_empty() {
-                        info.insert_sst_level = delta.level_idx;
+                if let GroupDelta::IntraLevel(intra_level) = group_delta {
+                    if !intra_level.inserted_table_infos.is_empty() {
+                        info.insert_sst_level = intra_level.level_idx;
                         info.insert_sst_infos
-                            .extend(delta.inserted_table_infos.iter().cloned());
+                            .extend(intra_level.inserted_table_infos.iter().cloned());
                     }
-                    if !delta.removed_table_ids.is_empty() {
-                        for id in &delta.removed_table_ids {
-                            if delta.level_idx == 0 {
+                    if !intra_level.removed_table_ids.is_empty() {
+                        for id in &intra_level.removed_table_ids {
+                            if intra_level.level_idx == 0 {
                                 removed_l0_ssts.insert(*id);
                             } else {
-                                removed_ssts.entry(delta.level_idx).or_default().insert(*id);
+                                removed_ssts
+                                    .entry(intra_level.level_idx)
+                                    .or_default()
+                                    .insert(*id);
                             }
                         }
                     }
@@ -438,14 +513,14 @@ impl HummockVersion {
             }
 
             let group = self.levels.get(group_id).unwrap();
-            for l0_sub_level in &group.get_level0().sub_levels {
+            for l0_sub_level in &group.level0().sub_levels {
                 for sst_info in &l0_sub_level.table_infos {
                     if removed_l0_ssts.remove(&sst_info.sst_id) {
                         info.delete_sst_object_ids.push(sst_info.object_id);
                     }
                 }
             }
-            for level in group.get_levels() {
+            for level in &group.levels {
                 if let Some(mut removed_level_ssts) = removed_ssts.remove(&level.level_idx) {
                     for sst_info in &level.table_infos {
                         if removed_level_ssts.remove(&sst_info.sst_id) {
@@ -520,7 +595,6 @@ impl HummockVersion {
                     *compaction_group_id,
                     member_table_ids,
                     group_construct.get_new_sst_start_id(),
-                    group_construct.version < CompatibilityVersion::NoTrivialSplit as _,
                 );
             } else if let Some(group_change) = &summary.group_table_change {
                 // TODO: may deprecate this branch? This enum variant is not created anywhere
@@ -535,7 +609,6 @@ impl HummockVersion {
                     group_change.target_group_id,
                     HashSet::from_iter(group_change.table_ids.clone()),
                     group_change.new_sst_start_id,
-                    group_change.version() == CompatibilityVersion::VersionUnspecified,
                 );
 
                 let levels = self
@@ -584,29 +657,35 @@ impl HummockVersion {
                 let GroupDeltasSummary {
                     delete_sst_levels,
                     delete_sst_ids_set,
-                    insert_sst_level_id,
-                    insert_sub_level_id,
-                    insert_table_infos,
                     ..
                 } = summary;
-                assert!(
-                    insert_sst_level_id == 0 || insert_table_infos.is_empty(),
-                    "we should only add to L0 when we commit an epoch. Inserting into {} {:?}",
-                    insert_sst_level_id,
-                    insert_table_infos
-                );
+
                 assert!(
                     delete_sst_levels.is_empty() && delete_sst_ids_set.is_empty() || has_destroy,
                     "no sst should be deleted when committing an epoch"
                 );
-                if !insert_table_infos.is_empty() {
-                    insert_new_sub_level(
-                        levels.l0.as_mut().unwrap(),
-                        insert_sub_level_id,
-                        LevelType::Overlapping,
-                        insert_table_infos,
-                        None,
-                    );
+                for group_delta in &group_deltas.group_deltas {
+                    if let GroupDelta::IntraLevel(IntraLevelDelta {
+                        level_idx,
+                        l0_sub_level_id,
+                        inserted_table_infos,
+                        ..
+                    }) = group_delta
+                    {
+                        assert_eq!(
+                            *level_idx, 0,
+                            "we should only add to L0 when we commit an epoch."
+                        );
+                        if !inserted_table_infos.is_empty() {
+                            insert_new_sub_level(
+                                &mut levels.l0,
+                                *l0_sub_level_id,
+                                PbLevelType::Overlapping,
+                                inserted_table_infos.clone(),
+                                None,
+                            );
+                        }
+                    }
                 }
             } else {
                 // `max_committed_epoch` is not changed. The delta is caused by compaction.
@@ -622,7 +701,7 @@ impl HummockVersion {
         }
         self.id = version_delta.id;
         self.max_committed_epoch = version_delta.max_committed_epoch;
-        self.set_safe_epoch(version_delta.safe_epoch);
+        self.set_safe_epoch(version_delta.visible_table_safe_epoch());
 
         // apply to table watermark
 
@@ -645,20 +724,22 @@ impl HummockVersion {
             }
         }
         for (table_id, table_watermarks) in &self.table_watermarks {
-            if let Some(table_delta) = version_delta.state_table_info_delta.get(table_id)
-                && let Some(Some(prev_table)) = changed_table_info.get(table_id)
-                && table_delta.safe_epoch > prev_table.safe_epoch
+            let safe_epoch = if let Some(state_table_info) =
+                self.state_table_info.info().get(table_id)
+                && let Some((oldest_epoch, _)) = table_watermarks.watermarks.first()
+                && state_table_info.safe_epoch > *oldest_epoch
             {
                 // safe epoch has progressed, need further clear.
+                state_table_info.safe_epoch
             } else {
-                // safe epoch not progressed. No need to truncate
+                // safe epoch not progressed or the table has been removed. No need to truncate
                 continue;
-            }
+            };
             let table_watermarks = modified_table_watermarks
                 .entry(*table_id)
                 .or_insert_with(|| Some((**table_watermarks).clone()));
             if let Some(table_watermarks) = table_watermarks {
-                table_watermarks.clear_stale_epoch_watermark(version_delta.safe_epoch);
+                table_watermarks.clear_stale_epoch_watermark(safe_epoch);
             }
         }
         // apply the staging table watermark to hummock version
@@ -728,14 +809,14 @@ impl HummockVersion {
         let mut ret: BTreeMap<_, _> = BTreeMap::new();
         for (compaction_group_id, group) in &self.levels {
             let mut levels = vec![];
-            levels.extend(group.l0.as_ref().unwrap().sub_levels.iter());
+            levels.extend(group.l0.sub_levels.iter());
             levels.extend(group.levels.iter());
             for level in levels {
                 for table_info in &level.table_infos {
                     if table_info.sst_id == table_info.object_id {
                         continue;
                     }
-                    let object_id = table_info.get_object_id();
+                    let object_id = table_info.object_id;
                     let entry: &mut BranchedSstInfo = ret.entry(object_id).or_default();
                     entry
                         .entry(*compaction_group_id)
@@ -750,34 +831,6 @@ impl HummockVersion {
 
 #[easy_ext::ext(HummockLevelsExt)]
 impl Levels {
-    pub fn get_level0(&self) -> &OverlappingLevel {
-        self.l0.as_ref().unwrap()
-    }
-
-    pub fn get_level(&self, level_idx: usize) -> &Level {
-        &self.levels[level_idx - 1]
-    }
-
-    pub fn get_level_mut(&mut self, level_idx: usize) -> &mut Level {
-        &mut self.levels[level_idx - 1]
-    }
-
-    pub fn is_last_level(&self, level_idx: u32) -> bool {
-        self.levels
-            .last()
-            .as_ref()
-            .map_or(false, |level| level.level_idx == level_idx)
-    }
-
-    pub fn count_ssts(&self) -> usize {
-        self.get_level0()
-            .get_sub_levels()
-            .iter()
-            .chain(self.get_levels().iter())
-            .map(|level| level.get_table_infos().len())
-            .sum()
-    }
-
     pub fn apply_compact_ssts(
         &mut self,
         summary: GroupDeltasSummary,
@@ -814,7 +867,7 @@ impl Levels {
         }
         for level_idx in &delete_sst_levels {
             if *level_idx == 0 {
-                for level in &mut self.l0.as_mut().unwrap().sub_levels {
+                for level in &mut self.l0.sub_levels {
                     level_delete_ssts(level, &delete_sst_ids_set);
                 }
             } else {
@@ -825,7 +878,7 @@ impl Levels {
 
         if !insert_table_infos.is_empty() {
             if insert_sst_level_id == 0 {
-                let l0 = self.l0.as_mut().unwrap();
+                let l0 = &mut self.l0;
                 let index = l0
                     .sub_levels
                     .partition_point(|level| level.sub_level_id < insert_sub_level_id);
@@ -866,22 +919,16 @@ impl Levels {
         }
         if delete_sst_levels.iter().any(|level_id| *level_id == 0) {
             self.l0
-                .as_mut()
-                .unwrap()
                 .sub_levels
                 .retain(|level| !level.table_infos.is_empty());
-            self.l0.as_mut().unwrap().total_file_size = self
+            self.l0.total_file_size = self
                 .l0
-                .as_mut()
-                .unwrap()
                 .sub_levels
                 .iter()
                 .map(|level| level.total_file_size)
                 .sum::<u64>();
-            self.l0.as_mut().unwrap().uncompressed_file_size = self
+            self.l0.uncompressed_file_size = self
                 .l0
-                .as_mut()
-                .unwrap()
                 .sub_levels
                 .iter()
                 .map(|level| level.uncompressed_file_size)
@@ -896,7 +943,7 @@ impl Levels {
     ) -> bool {
         for level_idx in delete_sst_levels {
             if *level_idx == 0 {
-                for level in &self.l0.as_ref().unwrap().sub_levels {
+                for level in &self.l0.sub_levels {
                     level.table_infos.iter().for_each(|table| {
                         delete_sst_ids_set.remove(&table.sst_id);
                     });
@@ -920,7 +967,7 @@ pub fn build_initial_compaction_group_levels(
     for l in 0..compaction_config.get_max_level() {
         levels.push(Level {
             level_idx: (l + 1) as u32,
-            level_type: LevelType::Nonoverlapping as i32,
+            level_type: PbLevelType::Nonoverlapping,
             table_infos: vec![],
             total_file_size: 0,
             sub_level_id: 0,
@@ -931,11 +978,11 @@ pub fn build_initial_compaction_group_levels(
     #[expect(deprecated)] // for backward-compatibility of previous hummock version delta
     Levels {
         levels,
-        l0: Some(OverlappingLevel {
+        l0: OverlappingLevel {
             sub_levels: vec![],
             total_file_size: 0,
             uncompressed_file_size: 0,
-        }),
+        },
         group_id,
         parent_group_id: StaticCompactionGroupId::NewCompactionGroup as _,
         member_table_ids: vec![],
@@ -944,7 +991,6 @@ pub fn build_initial_compaction_group_levels(
 
 fn split_sst_info_for_level(
     member_table_ids: &HashSet<u32>,
-    allow_trivial_split: bool,
     level: &mut Level,
     new_sst_id: &mut u64,
 ) -> Vec<SstableInfo> {
@@ -959,22 +1005,8 @@ fn split_sst_info_for_level(
             .cloned()
             .collect_vec();
         if !removed_table_ids.is_empty() {
-            let is_trivial =
-                allow_trivial_split && removed_table_ids.len() == sst_info.table_ids.len();
-            let mut branch_table_info = sst_info.clone();
-            branch_table_info.sst_id = *new_sst_id;
-            *new_sst_id += 1;
-            if is_trivial {
-                // This is a compatibility design. we only clear the table-ids for files which would
-                // be removed in later code. In the version-delta generated by new
-                // version meta-service, there will be no trivial split, and we will create
-                // a reference for every sstable split to two groups.
-                sst_info.table_ids.clear();
-            } else {
-                sst_info.sst_id = *new_sst_id;
-                *new_sst_id += 1;
-            }
-            insert_table_infos.push(branch_table_info);
+            let branch_sst = split_sst(sst_info, new_sst_id);
+            insert_table_infos.push(branch_sst);
         }
     }
     insert_table_infos
@@ -1006,8 +1038,6 @@ pub fn get_compaction_group_ssts(
     let group_levels = version.get_compaction_group_levels(group_id);
     group_levels
         .l0
-        .as_ref()
-        .unwrap()
         .sub_levels
         .iter()
         .rev()
@@ -1016,16 +1046,16 @@ pub fn get_compaction_group_ssts(
             level
                 .table_infos
                 .iter()
-                .map(|table_info| (table_info.get_object_id(), table_info.get_sst_id()))
+                .map(|table_info| (table_info.object_id, table_info.sst_id))
         })
 }
 
 pub fn new_sub_level(
     sub_level_id: u64,
-    level_type: LevelType,
+    level_type: PbLevelType,
     table_infos: Vec<SstableInfo>,
 ) -> Level {
-    if level_type == LevelType::Nonoverlapping {
+    if level_type == PbLevelType::Nonoverlapping {
         debug_assert!(
             can_concat(&table_infos),
             "sst of non-overlapping level is not concat-able: {:?}",
@@ -1039,7 +1069,7 @@ pub fn new_sub_level(
         .sum();
     Level {
         level_idx: 0,
-        level_type: level_type as i32,
+        level_type,
         table_infos,
         total_file_size,
         sub_level_id,
@@ -1062,14 +1092,10 @@ pub fn add_ssts_to_sub_level(
     l0.sub_levels[sub_level_idx]
         .table_infos
         .extend(insert_table_infos);
-    if l0.sub_levels[sub_level_idx].level_type == LevelType::Nonoverlapping as i32 {
+    if l0.sub_levels[sub_level_idx].level_type == PbLevelType::Nonoverlapping {
         l0.sub_levels[sub_level_idx]
             .table_infos
-            .sort_by(|sst1, sst2| {
-                let a = sst1.key_range.as_ref().unwrap();
-                let b = sst2.key_range.as_ref().unwrap();
-                a.compare(b)
-            });
+            .sort_by(|sst1, sst2| sst1.key_range.cmp(&sst2.key_range));
         assert!(
             can_concat(&l0.sub_levels[sub_level_idx].table_infos),
             "sstable ids: {:?}",
@@ -1086,7 +1112,7 @@ pub fn add_ssts_to_sub_level(
 pub fn insert_new_sub_level(
     l0: &mut OverlappingLevel,
     insert_sub_level_id: u64,
-    level_type: LevelType,
+    level_type: PbLevelType,
     insert_table_infos: Vec<SstableInfo>,
     sub_level_insert_hint: Option<usize>,
 ) {
@@ -1111,11 +1137,11 @@ pub fn insert_new_sub_level(
     {
         if insert_pos > 0 {
             if let Some(smaller_level) = l0.sub_levels.get(insert_pos - 1) {
-                debug_assert!(smaller_level.get_sub_level_id() < insert_sub_level_id);
+                debug_assert!(smaller_level.sub_level_id < insert_sub_level_id);
             }
         }
         if let Some(larger_level) = l0.sub_levels.get(insert_pos) {
-            debug_assert!(larger_level.get_sub_level_id() > insert_sub_level_id);
+            debug_assert!(larger_level.sub_level_id > insert_sub_level_id);
         }
     }
     // All files will be committed in one new Overlapping sub-level and become
@@ -1160,13 +1186,11 @@ fn level_insert_ssts(operand: &mut Level, insert_table_infos: Vec<SstableInfo>)
         .map(|sst| sst.uncompressed_file_size)
         .sum::<u64>();
     operand.table_infos.extend(insert_table_infos);
-    operand.table_infos.sort_by(|sst1, sst2| {
-        let a = sst1.key_range.as_ref().unwrap();
-        let b = sst2.key_range.as_ref().unwrap();
-        a.compare(b)
-    });
-    if operand.level_type == LevelType::Overlapping as i32 {
-        operand.level_type = LevelType::Nonoverlapping as i32;
+    operand
+        .table_infos
+        .sort_by(|sst1, sst2| sst1.key_range.cmp(&sst2.key_range));
+    if operand.level_type == PbLevelType::Overlapping {
+        operand.level_type = PbLevelType::Nonoverlapping;
     }
     assert!(
         can_concat(&operand.table_infos),
@@ -1184,16 +1208,11 @@ pub fn object_size_map(version: &HummockVersion) -> HashMap<HummockSstableObject
         .levels
         .values()
         .flat_map(|cg| {
-            cg.get_level0()
-                .get_sub_levels()
+            cg.level0()
+                .sub_levels
                 .iter()
-                .chain(cg.get_levels().iter())
-                .flat_map(|level| {
-                    level
-                        .get_table_infos()
-                        .iter()
-                        .map(|t| (t.object_id, t.file_size))
-                })
+                .chain(cg.levels.iter())
+                .flat_map(|level| level.table_infos.iter().map(|t| (t.object_id, t.file_size)))
         })
         .collect()
 }
@@ -1233,12 +1252,11 @@ pub fn validate_version(version: &HummockVersion) -> Vec<String> {
                 if level.table_infos.is_empty() {
                     res.push(format!("{}: empty level", level_identifier));
                 }
-            } else if level.level_type() != PbLevelType::Nonoverlapping {
+            } else if level.level_type != PbLevelType::Nonoverlapping {
                 // Ensure non-L0 level is non-overlapping level
                 res.push(format!(
                     "{}: level type {:?} is not non-overlapping",
-                    level_identifier,
-                    level.level_type()
+                    level_identifier, level.level_type
                 ));
             }
 
@@ -1261,13 +1279,11 @@ pub fn validate_version(version: &HummockVersion) -> Vec<String> {
                 }
 
                 // Ensure SSTs in non-overlapping level have non-overlapping key range
-                if level.level_type() == PbLevelType::Nonoverlapping {
+                if level.level_type == PbLevelType::Nonoverlapping {
                     if let Some(prev) = prev_table_info.take() {
                         if prev
                             .key_range
-                            .as_ref()
-                            .unwrap()
-                            .compare_right_with(&table_info.key_range.as_ref().unwrap().left)
+                            .compare_right_with(&table_info.key_range.left)
                             != Ordering::Less
                         {
                             res.push(format!(
@@ -1281,22 +1297,19 @@ pub fn validate_version(version: &HummockVersion) -> Vec<String> {
             }
         };
 
-        if let Some(l0) = &levels.l0 {
-            let mut prev_sub_level_id = u64::MAX;
-            for sub_level in &l0.sub_levels {
-                // Ensure sub_level_id is sorted and unique
-                if sub_level.sub_level_id >= prev_sub_level_id {
-                    res.push(format!(
-                        "GROUP {} LEVEL 0: sub_level_id {} >= prev_sub_level {}",
-                        group_id, sub_level.level_idx, prev_sub_level_id
-                    ));
-                }
-                prev_sub_level_id = sub_level.sub_level_id;
-
-                validate_level(*group_id, 0, sub_level, &mut res);
+        let l0 = &levels.l0;
+        let mut prev_sub_level_id = u64::MAX;
+        for sub_level in &l0.sub_levels {
+            // Ensure sub_level_id is sorted and unique
+            if sub_level.sub_level_id >= prev_sub_level_id {
+                res.push(format!(
+                    "GROUP {} LEVEL 0: sub_level_id {} >= prev_sub_level {}",
+                    group_id, sub_level.level_idx, prev_sub_level_id
+                ));
             }
-        } else {
-            res.push(format!("GROUP {}: level0 not exist", group_id));
+            prev_sub_level_id = sub_level.sub_level_id;
+
+            validate_level(*group_id, 0, sub_level, &mut res);
         }
 
         for idx in 1..=levels.levels.len() {
@@ -1306,20 +1319,27 @@ pub fn validate_version(version: &HummockVersion) -> Vec<String> {
     res
 }
 
+pub fn split_sst(sst_info: &mut SstableInfo, new_sst_id: &mut u64) -> SstableInfo {
+    let mut branch_table_info = sst_info.clone();
+    branch_table_info.sst_id = *new_sst_id;
+    sst_info.sst_id = *new_sst_id + 1;
+    *new_sst_id += 1;
+
+    branch_table_info
+}
+
 #[cfg(test)]
 mod tests {
     use std::collections::HashMap;
 
-    use risingwave_pb::hummock::group_delta::DeltaType;
-    use risingwave_pb::hummock::hummock_version::Levels;
-    use risingwave_pb::hummock::hummock_version_delta::GroupDeltas;
-    use risingwave_pb::hummock::{
-        CompactionConfig, GroupConstruct, GroupDelta, GroupDestroy, IntraLevelDelta, Level,
-        LevelType, OverlappingLevel, SstableInfo,
-    };
+    use risingwave_pb::hummock::{CompactionConfig, GroupConstruct, GroupDestroy, LevelType};
 
     use crate::compaction_group::hummock_version_ext::build_initial_compaction_group_levels;
-    use crate::version::{HummockVersion, HummockVersionDelta};
+    use crate::level::{Level, Levels, OverlappingLevel};
+    use crate::sstable_info::SstableInfo;
+    use crate::version::{
+        GroupDelta, GroupDeltas, HummockVersion, HummockVersionDelta, IntraLevelDelta,
+    };
 
     #[test]
     fn test_get_sst_object_ids() {
@@ -1329,11 +1349,11 @@ mod tests {
             0,
             Levels {
                 levels: vec![],
-                l0: Some(OverlappingLevel {
+                l0: OverlappingLevel {
                     sub_levels: vec![],
                     total_file_size: 0,
                     uncompressed_file_size: 0,
-                }),
+                },
                 ..Default::default()
             },
         )]);
@@ -1345,8 +1365,6 @@ mod tests {
             .get_mut(&0)
             .unwrap()
             .l0
-            .as_mut()
-            .unwrap()
             .sub_levels
             .push(Level {
                 table_infos: vec![SstableInfo {
@@ -1396,50 +1414,46 @@ mod tests {
                 ),
             ),
         ]);
-        let version_delta = HummockVersionDelta {
-            id: 1,
-            group_deltas: HashMap::from_iter([
-                (
-                    2,
-                    GroupDeltas {
-                        group_deltas: vec![GroupDelta {
-                            delta_type: Some(DeltaType::GroupConstruct(GroupConstruct {
-                                group_config: Some(CompactionConfig {
-                                    max_level: 6,
-                                    ..Default::default()
-                                }),
-                                ..Default::default()
-                            })),
-                        }],
-                    },
-                ),
-                (
-                    0,
-                    GroupDeltas {
-                        group_deltas: vec![GroupDelta {
-                            delta_type: Some(DeltaType::GroupDestroy(GroupDestroy {})),
-                        }],
-                    },
-                ),
-                (
-                    1,
-                    GroupDeltas {
-                        group_deltas: vec![GroupDelta {
-                            delta_type: Some(DeltaType::IntraLevel(IntraLevelDelta {
-                                level_idx: 1,
-                                inserted_table_infos: vec![SstableInfo {
-                                    object_id: 1,
-                                    sst_id: 1,
-                                    ..Default::default()
-                                }],
-                                ..Default::default()
-                            })),
+        let mut version_delta = HummockVersionDelta::default();
+        version_delta.id = 1;
+        version_delta.group_deltas = HashMap::from_iter([
+            (
+                2,
+                GroupDeltas {
+                    group_deltas: vec![GroupDelta::GroupConstruct(GroupConstruct {
+                        group_config: Some(CompactionConfig {
+                            max_level: 6,
+                            ..Default::default()
+                        }),
+                        ..Default::default()
+                    })],
+                },
+            ),
+            (
+                0,
+                GroupDeltas {
+                    group_deltas: vec![GroupDelta::GroupDestroy(GroupDestroy {})],
+                },
+            ),
+            (
+                1,
+                GroupDeltas {
+                    group_deltas: vec![GroupDelta::IntraLevel(IntraLevelDelta::new(
+                        1,
+                        0,
+                        vec![],
+                        vec![SstableInfo {
+                            object_id: 1,
+                            sst_id: 1,
+                            ..Default::default()
                         }],
-                    },
-                ),
-            ]),
-            ..Default::default()
-        };
+                        0,
+                    ))],
+                },
+            ),
+        ]);
+        let version_delta = version_delta;
+
         version.apply_version_delta(&version_delta);
         let mut cg1 = build_initial_compaction_group_levels(
             1,
@@ -1450,7 +1464,7 @@ mod tests {
         );
         cg1.levels[0] = Level {
             level_idx: 1,
-            level_type: LevelType::Nonoverlapping as i32,
+            level_type: LevelType::Nonoverlapping,
             table_infos: vec![SstableInfo {
                 object_id: 1,
                 sst_id: 1,
diff --git a/src/storage/hummock_sdk/src/key_range.rs b/src/storage/hummock_sdk/src/key_range.rs
index 95359e8a54974..2fa3d69f49342 100644
--- a/src/storage/hummock_sdk/src/key_range.rs
+++ b/src/storage/hummock_sdk/src/key_range.rs
@@ -52,6 +52,11 @@ impl KeyRange {
     fn end_bound_inf(&self) -> bool {
         self.right.is_empty()
     }
+
+    #[inline]
+    pub fn inf_key_range(&self) -> bool {
+        self.start_bound_inf() && self.end_bound_inf()
+    }
 }
 
 pub trait KeyRangeCommon {
diff --git a/src/storage/hummock_sdk/src/level.rs b/src/storage/hummock_sdk/src/level.rs
new file mode 100644
index 0000000000000..c7db09e69e76d
--- /dev/null
+++ b/src/storage/hummock_sdk/src/level.rs
@@ -0,0 +1,388 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::mem::size_of;
+
+use itertools::Itertools;
+use risingwave_pb::hummock::hummock_version::PbLevels;
+use risingwave_pb::hummock::{
+    PbInputLevel, PbLevel, PbLevelType, PbOverlappingLevel, PbSstableInfo,
+};
+
+use crate::sstable_info::SstableInfo;
+
+#[derive(Debug, Clone, PartialEq, Default)]
+pub struct OverlappingLevel {
+    pub sub_levels: Vec<Level>,
+    pub total_file_size: u64,
+    pub uncompressed_file_size: u64,
+}
+
+impl From<&PbOverlappingLevel> for OverlappingLevel {
+    fn from(pb_overlapping_level: &PbOverlappingLevel) -> Self {
+        Self {
+            sub_levels: pb_overlapping_level
+                .sub_levels
+                .iter()
+                .map(Level::from)
+                .collect_vec(),
+            total_file_size: pb_overlapping_level.total_file_size,
+            uncompressed_file_size: pb_overlapping_level.uncompressed_file_size,
+        }
+    }
+}
+
+impl From<&OverlappingLevel> for PbOverlappingLevel {
+    fn from(overlapping_level: &OverlappingLevel) -> Self {
+        Self {
+            sub_levels: overlapping_level
+                .sub_levels
+                .iter()
+                .map(|pb_level| pb_level.into())
+                .collect_vec(),
+            total_file_size: overlapping_level.total_file_size,
+            uncompressed_file_size: overlapping_level.uncompressed_file_size,
+        }
+    }
+}
+
+impl From<OverlappingLevel> for PbOverlappingLevel {
+    fn from(overlapping_level: OverlappingLevel) -> Self {
+        Self {
+            sub_levels: overlapping_level
+                .sub_levels
+                .into_iter()
+                .map(|pb_level| pb_level.into())
+                .collect_vec(),
+            total_file_size: overlapping_level.total_file_size,
+            uncompressed_file_size: overlapping_level.uncompressed_file_size,
+        }
+    }
+}
+
+impl From<PbOverlappingLevel> for OverlappingLevel {
+    fn from(pb_overlapping_level: PbOverlappingLevel) -> Self {
+        Self {
+            sub_levels: pb_overlapping_level
+                .sub_levels
+                .into_iter()
+                .map(Level::from)
+                .collect_vec(),
+            total_file_size: pb_overlapping_level.total_file_size,
+            uncompressed_file_size: pb_overlapping_level.uncompressed_file_size,
+        }
+    }
+}
+
+impl OverlappingLevel {
+    pub fn estimated_encode_len(&self) -> usize {
+        self.sub_levels
+            .iter()
+            .map(|level| level.estimated_encode_len())
+            .sum::<usize>()
+            + size_of::<u64>()
+            + size_of::<u64>()
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Default)]
+pub struct Level {
+    pub level_idx: u32,
+    pub level_type: PbLevelType,
+    pub table_infos: Vec<SstableInfo>,
+    pub total_file_size: u64,
+    pub sub_level_id: u64,
+    pub uncompressed_file_size: u64,
+    pub vnode_partition_count: u32,
+}
+
+impl From<&PbLevel> for Level {
+    fn from(pb_level: &PbLevel) -> Self {
+        Self {
+            level_idx: pb_level.level_idx,
+            level_type: PbLevelType::try_from(pb_level.level_type).unwrap(),
+            table_infos: pb_level
+                .table_infos
+                .iter()
+                .map(SstableInfo::from)
+                .collect_vec(),
+            total_file_size: pb_level.total_file_size,
+            sub_level_id: pb_level.sub_level_id,
+            uncompressed_file_size: pb_level.uncompressed_file_size,
+            vnode_partition_count: pb_level.vnode_partition_count,
+        }
+    }
+}
+
+impl From<&Level> for PbLevel {
+    fn from(level: &Level) -> Self {
+        Self {
+            level_idx: level.level_idx,
+            level_type: level.level_type.into(),
+            table_infos: level
+                .table_infos
+                .iter()
+                .map(PbSstableInfo::from)
+                .collect_vec(),
+            total_file_size: level.total_file_size,
+            sub_level_id: level.sub_level_id,
+            uncompressed_file_size: level.uncompressed_file_size,
+            vnode_partition_count: level.vnode_partition_count,
+        }
+    }
+}
+
+impl From<Level> for PbLevel {
+    fn from(level: Level) -> Self {
+        Self {
+            level_idx: level.level_idx,
+            level_type: level.level_type.into(),
+            table_infos: level
+                .table_infos
+                .into_iter()
+                .map(PbSstableInfo::from)
+                .collect_vec(),
+            total_file_size: level.total_file_size,
+            sub_level_id: level.sub_level_id,
+            uncompressed_file_size: level.uncompressed_file_size,
+            vnode_partition_count: level.vnode_partition_count,
+        }
+    }
+}
+
+impl From<PbLevel> for Level {
+    fn from(pb_level: PbLevel) -> Self {
+        Self {
+            level_idx: pb_level.level_idx,
+            level_type: PbLevelType::try_from(pb_level.level_type).unwrap(),
+            table_infos: pb_level
+                .table_infos
+                .into_iter()
+                .map(SstableInfo::from)
+                .collect_vec(),
+            total_file_size: pb_level.total_file_size,
+            sub_level_id: pb_level.sub_level_id,
+            uncompressed_file_size: pb_level.uncompressed_file_size,
+            vnode_partition_count: pb_level.vnode_partition_count,
+        }
+    }
+}
+
+impl Level {
+    pub fn estimated_encode_len(&self) -> usize {
+        size_of::<u32>()
+            + size_of::<u32>()
+            + self
+                .table_infos
+                .iter()
+                .map(|sst| sst.estimated_encode_len())
+                .sum::<usize>()
+            + size_of::<u64>()
+            + size_of::<u64>()
+            + size_of::<u64>()
+            + size_of::<u32>()
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Default)]
+pub struct Levels {
+    pub levels: Vec<Level>,
+    pub l0: OverlappingLevel,
+    pub group_id: u64,
+    pub parent_group_id: u64,
+
+    #[deprecated]
+    pub member_table_ids: Vec<u32>,
+}
+
+impl Levels {
+    pub fn level0(&self) -> &OverlappingLevel {
+        &self.l0
+    }
+
+    pub fn get_level(&self, level_idx: usize) -> &Level {
+        &self.levels[level_idx - 1]
+    }
+
+    pub fn get_level_mut(&mut self, level_idx: usize) -> &mut Level {
+        &mut self.levels[level_idx - 1]
+    }
+
+    pub fn is_last_level(&self, level_idx: u32) -> bool {
+        self.levels
+            .last()
+            .as_ref()
+            .map_or(false, |level| level.level_idx == level_idx)
+    }
+
+    pub fn count_ssts(&self) -> usize {
+        self.level0()
+            .sub_levels
+            .iter()
+            .chain(self.levels.iter())
+            .map(|level| level.table_infos.len())
+            .sum()
+    }
+}
+
+impl Levels {
+    pub fn from_protobuf(pb_levels: &PbLevels) -> Self {
+        Self::from(pb_levels)
+    }
+
+    pub fn to_protobuf(&self) -> PbLevels {
+        self.into()
+    }
+
+    pub fn estimated_encode_len(&self) -> usize {
+        let mut basic = self
+            .levels
+            .iter()
+            .map(|level| level.estimated_encode_len())
+            .sum::<usize>()
+            + size_of::<u64>()
+            + size_of::<u64>()
+            + size_of::<u32>();
+        basic += self.l0.estimated_encode_len();
+
+        basic
+    }
+}
+
+impl From<&PbLevels> for Levels {
+    #[expect(deprecated)]
+    fn from(pb_levels: &PbLevels) -> Self {
+        Self {
+            l0: OverlappingLevel::from(pb_levels.l0.as_ref().unwrap()),
+            levels: pb_levels.levels.iter().map(Level::from).collect_vec(),
+            group_id: pb_levels.group_id,
+            parent_group_id: pb_levels.parent_group_id,
+            member_table_ids: pb_levels.member_table_ids.clone(),
+        }
+    }
+}
+
+impl From<&Levels> for PbLevels {
+    #[expect(deprecated)]
+    fn from(levels: &Levels) -> Self {
+        Self {
+            l0: Some((&levels.l0).into()),
+            levels: levels.levels.iter().map(PbLevel::from).collect_vec(),
+            group_id: levels.group_id,
+            parent_group_id: levels.parent_group_id,
+            member_table_ids: levels.member_table_ids.clone(),
+        }
+    }
+}
+
+impl From<PbLevels> for Levels {
+    #[expect(deprecated)]
+    fn from(pb_levels: PbLevels) -> Self {
+        Self {
+            l0: OverlappingLevel::from(pb_levels.l0.as_ref().unwrap()),
+            levels: pb_levels.levels.into_iter().map(Level::from).collect_vec(),
+            group_id: pb_levels.group_id,
+            parent_group_id: pb_levels.parent_group_id,
+            member_table_ids: pb_levels.member_table_ids.clone(),
+        }
+    }
+}
+
+impl From<Levels> for PbLevels {
+    fn from(levels: Levels) -> Self {
+        #[expect(deprecated)]
+        Self {
+            l0: Some(levels.l0.into()),
+            levels: levels.levels.into_iter().map(PbLevel::from).collect_vec(),
+            group_id: levels.group_id,
+            parent_group_id: levels.parent_group_id,
+            member_table_ids: levels.member_table_ids.clone(),
+        }
+    }
+}
+
+#[derive(Clone, PartialEq, Default, Debug)]
+pub struct InputLevel {
+    pub level_idx: u32,
+    pub level_type: PbLevelType,
+    pub table_infos: Vec<SstableInfo>,
+}
+
+impl InputLevel {
+    pub fn estimated_encode_len(&self) -> usize {
+        size_of::<u32>()
+            + size_of::<i32>()
+            + self
+                .table_infos
+                .iter()
+                .map(|sst| sst.estimated_encode_len())
+                .sum::<usize>()
+    }
+}
+
+impl From<PbInputLevel> for InputLevel {
+    fn from(pb_input_level: PbInputLevel) -> Self {
+        Self {
+            level_idx: pb_input_level.level_idx,
+            level_type: PbLevelType::try_from(pb_input_level.level_type).unwrap(),
+            table_infos: pb_input_level
+                .table_infos
+                .into_iter()
+                .map(SstableInfo::from)
+                .collect_vec(),
+        }
+    }
+}
+
+impl From<&PbInputLevel> for InputLevel {
+    fn from(pb_input_level: &PbInputLevel) -> Self {
+        Self {
+            level_idx: pb_input_level.level_idx,
+            level_type: PbLevelType::try_from(pb_input_level.level_type).unwrap(),
+            table_infos: pb_input_level
+                .table_infos
+                .iter()
+                .map(SstableInfo::from)
+                .collect_vec(),
+        }
+    }
+}
+
+impl From<InputLevel> for PbInputLevel {
+    fn from(input_level: InputLevel) -> Self {
+        Self {
+            level_idx: input_level.level_idx,
+            level_type: input_level.level_type.into(),
+            table_infos: input_level
+                .table_infos
+                .into_iter()
+                .map(|sst| sst.into())
+                .collect_vec(),
+        }
+    }
+}
+
+impl From<&InputLevel> for PbInputLevel {
+    fn from(input_level: &InputLevel) -> Self {
+        Self {
+            level_idx: input_level.level_idx,
+            level_type: input_level.level_type.into(),
+            table_infos: input_level
+                .table_infos
+                .iter()
+                .map(|sst| sst.into())
+                .collect_vec(),
+        }
+    }
+}
diff --git a/src/storage/hummock_sdk/src/lib.rs b/src/storage/hummock_sdk/src/lib.rs
index 97e1a334dcf98..c894ecf184128 100644
--- a/src/storage/hummock_sdk/src/lib.rs
+++ b/src/storage/hummock_sdk/src/lib.rs
@@ -22,7 +22,6 @@
 #![feature(is_sorted)]
 #![feature(let_chains)]
 #![feature(btree_cursors)]
-#![feature(lazy_cell)]
 
 mod key_cmp;
 use std::cmp::Ordering;
@@ -31,20 +30,23 @@ use std::collections::HashMap;
 pub use key_cmp::*;
 use risingwave_common::util::epoch::EPOCH_SPILL_TIME_MASK;
 use risingwave_pb::common::{batch_query_epoch, BatchQueryEpoch};
-use risingwave_pb::hummock::SstableInfo;
+use sstable_info::SstableInfo;
 
-use crate::compaction_group::StaticCompactionGroupId;
 use crate::key_range::KeyRangeCommon;
-use crate::table_stats::{to_prost_table_stats_map, PbTableStatsMap, TableStatsMap};
+use crate::table_stats::TableStatsMap;
 
 pub mod change_log;
 pub mod compact;
+pub mod compact_task;
 pub mod compaction_group;
 pub mod key;
 pub mod key_range;
+pub mod level;
 pub mod prost_key_range;
+pub mod sstable_info;
 pub mod table_stats;
 pub mod table_watermark;
+pub mod time_travel;
 pub mod version;
 
 pub use compact::*;
@@ -106,42 +108,20 @@ pub struct SyncResult {
 
 #[derive(Debug, Clone)]
 pub struct LocalSstableInfo {
-    pub compaction_group_id: CompactionGroupId,
     pub sst_info: SstableInfo,
     pub table_stats: TableStatsMap,
 }
 
 impl LocalSstableInfo {
-    pub fn new(
-        compaction_group_id: CompactionGroupId,
-        sst_info: SstableInfo,
-        table_stats: TableStatsMap,
-    ) -> Self {
+    pub fn new(sst_info: SstableInfo, table_stats: TableStatsMap) -> Self {
         Self {
-            compaction_group_id,
             sst_info,
             table_stats,
         }
     }
 
-    pub fn with_compaction_group(
-        compaction_group_id: CompactionGroupId,
-        sst_info: SstableInfo,
-    ) -> Self {
-        Self::new(compaction_group_id, sst_info, TableStatsMap::default())
-    }
-
-    pub fn with_stats(sst_info: SstableInfo, table_stats: TableStatsMap) -> Self {
-        Self::new(
-            StaticCompactionGroupId::StateDefault as CompactionGroupId,
-            sst_info,
-            table_stats,
-        )
-    }
-
     pub fn for_test(sst_info: SstableInfo) -> Self {
         Self {
-            compaction_group_id: StaticCompactionGroupId::StateDefault as CompactionGroupId,
             sst_info,
             table_stats: Default::default(),
         }
@@ -152,47 +132,9 @@ impl LocalSstableInfo {
     }
 }
 
-#[derive(Debug, Clone)]
-pub struct ExtendedSstableInfo {
-    pub compaction_group_id: CompactionGroupId,
-    pub sst_info: SstableInfo,
-    pub table_stats: PbTableStatsMap,
-}
-
-impl ExtendedSstableInfo {
-    pub fn new(
-        compaction_group_id: CompactionGroupId,
-        sst_info: SstableInfo,
-        table_stats: PbTableStatsMap,
-    ) -> Self {
-        Self {
-            compaction_group_id,
-            sst_info,
-            table_stats,
-        }
-    }
-
-    pub fn with_compaction_group(
-        compaction_group_id: CompactionGroupId,
-        sst_info: SstableInfo,
-    ) -> Self {
-        Self::new(compaction_group_id, sst_info, PbTableStatsMap::default())
-    }
-}
-
-impl From<LocalSstableInfo> for ExtendedSstableInfo {
-    fn from(value: LocalSstableInfo) -> Self {
-        Self {
-            compaction_group_id: value.compaction_group_id,
-            sst_info: value.sst_info,
-            table_stats: to_prost_table_stats_map(value.table_stats),
-        }
-    }
-}
-
 impl PartialEq for LocalSstableInfo {
     fn eq(&self, other: &Self) -> bool {
-        self.compaction_group_id == other.compaction_group_id && self.sst_info == other.sst_info
+        self.sst_info == other.sst_info
     }
 }
 
@@ -207,6 +149,7 @@ pub enum HummockReadEpoch {
     NoWait(HummockEpoch),
     /// We don't need to wait epoch.
     Backup(HummockEpoch),
+    TimeTravel(HummockEpoch),
 }
 
 impl From<BatchQueryEpoch> for HummockReadEpoch {
@@ -215,6 +158,7 @@ impl From<BatchQueryEpoch> for HummockReadEpoch {
             batch_query_epoch::Epoch::Committed(epoch) => HummockReadEpoch::Committed(epoch),
             batch_query_epoch::Epoch::Current(epoch) => HummockReadEpoch::Current(epoch),
             batch_query_epoch::Epoch::Backup(epoch) => HummockReadEpoch::Backup(epoch),
+            batch_query_epoch::Epoch::TimeTravel(epoch) => HummockReadEpoch::TimeTravel(epoch),
         }
     }
 }
@@ -232,6 +176,7 @@ impl HummockReadEpoch {
             HummockReadEpoch::Current(epoch) => epoch,
             HummockReadEpoch::NoWait(epoch) => epoch,
             HummockReadEpoch::Backup(epoch) => epoch,
+            HummockReadEpoch::TimeTravel(epoch) => epoch,
         }
     }
 }
@@ -267,9 +212,7 @@ pub fn can_concat(ssts: &[SstableInfo]) -> bool {
     for i in 1..len {
         if ssts[i - 1]
             .key_range
-            .as_ref()
-            .unwrap()
-            .compare_right_with(&ssts[i].key_range.as_ref().unwrap().left)
+            .compare_right_with(&ssts[i].key_range.left)
             != Ordering::Less
         {
             return false;
diff --git a/src/storage/hummock_sdk/src/sstable_info.rs b/src/storage/hummock_sdk/src/sstable_info.rs
new file mode 100644
index 0000000000000..917c0d4835c31
--- /dev/null
+++ b/src/storage/hummock_sdk/src/sstable_info.rs
@@ -0,0 +1,198 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::mem::size_of;
+
+use risingwave_pb::hummock::{PbBloomFilterType, PbKeyRange, PbSstableInfo};
+
+use crate::key_range::KeyRange;
+
+#[derive(Debug, PartialEq, Clone, Default)]
+pub struct SstableInfo {
+    pub object_id: u64,
+    pub sst_id: u64,
+    pub key_range: KeyRange,
+    pub file_size: u64,
+    pub table_ids: Vec<u32>,
+    pub meta_offset: u64,
+    pub stale_key_count: u64,
+    pub total_key_count: u64,
+    pub min_epoch: u64,
+    pub max_epoch: u64,
+    pub uncompressed_file_size: u64,
+    pub range_tombstone_count: u64,
+    pub bloom_filter_kind: PbBloomFilterType,
+}
+
+impl SstableInfo {
+    pub fn estimated_encode_len(&self) -> usize {
+        let mut basic = size_of::<u64>() // object_id
+            + size_of::<u64>() // sstable_id
+            + size_of::<u64>() // file_size
+            + self.table_ids.len() * size_of::<u32>() // table_ids
+            + size_of::<u64>() // meta_offset
+            + size_of::<u64>() // stale_key_count
+            + size_of::<u64>() // total_key_count
+            + size_of::<u64>() // min_epoch
+            + size_of::<u64>() // max_epoch
+            + size_of::<u64>() // uncompressed_file_size
+            + size_of::<u64>() // range_tombstone_count
+            + size_of::<u32>(); // bloom_filter_kind
+        basic += self.key_range.left.len() + self.key_range.right.len() + size_of::<bool>();
+
+        basic
+    }
+
+    pub fn to_protobuf(&self) -> PbSstableInfo {
+        self.into()
+    }
+}
+
+impl From<PbSstableInfo> for SstableInfo {
+    fn from(pb_sstable_info: PbSstableInfo) -> Self {
+        Self {
+            object_id: pb_sstable_info.object_id,
+            sst_id: pb_sstable_info.sst_id,
+            key_range: {
+                // Due to the stripped key range, the key range may be `None`
+                if let Some(pb_keyrange) = pb_sstable_info.key_range {
+                    KeyRange {
+                        left: pb_keyrange.left.into(),
+                        right: pb_keyrange.right.into(),
+                        right_exclusive: pb_keyrange.right_exclusive,
+                    }
+                } else {
+                    KeyRange::inf()
+                }
+            },
+            file_size: pb_sstable_info.file_size,
+            table_ids: pb_sstable_info.table_ids.clone(),
+            meta_offset: pb_sstable_info.meta_offset,
+            stale_key_count: pb_sstable_info.stale_key_count,
+            total_key_count: pb_sstable_info.total_key_count,
+            min_epoch: pb_sstable_info.min_epoch,
+            max_epoch: pb_sstable_info.max_epoch,
+            uncompressed_file_size: pb_sstable_info.uncompressed_file_size,
+            range_tombstone_count: pb_sstable_info.range_tombstone_count,
+            bloom_filter_kind: PbBloomFilterType::try_from(pb_sstable_info.bloom_filter_kind)
+                .unwrap(),
+        }
+    }
+}
+
+impl From<&PbSstableInfo> for SstableInfo {
+    fn from(pb_sstable_info: &PbSstableInfo) -> Self {
+        Self {
+            object_id: pb_sstable_info.object_id,
+            sst_id: pb_sstable_info.sst_id,
+            key_range: {
+                if let Some(pb_keyrange) = &pb_sstable_info.key_range {
+                    KeyRange {
+                        left: pb_keyrange.left.clone().into(),
+                        right: pb_keyrange.right.clone().into(),
+                        right_exclusive: pb_keyrange.right_exclusive,
+                    }
+                } else {
+                    KeyRange::inf()
+                }
+            },
+            file_size: pb_sstable_info.file_size,
+            table_ids: pb_sstable_info.table_ids.clone(),
+            meta_offset: pb_sstable_info.meta_offset,
+            stale_key_count: pb_sstable_info.stale_key_count,
+            total_key_count: pb_sstable_info.total_key_count,
+            min_epoch: pb_sstable_info.min_epoch,
+            max_epoch: pb_sstable_info.max_epoch,
+            uncompressed_file_size: pb_sstable_info.uncompressed_file_size,
+            range_tombstone_count: pb_sstable_info.range_tombstone_count,
+            bloom_filter_kind: PbBloomFilterType::try_from(pb_sstable_info.bloom_filter_kind)
+                .unwrap(),
+        }
+    }
+}
+
+impl From<SstableInfo> for PbSstableInfo {
+    fn from(sstable_info: SstableInfo) -> Self {
+        PbSstableInfo {
+            object_id: sstable_info.object_id,
+            sst_id: sstable_info.sst_id,
+            key_range: {
+                let keyrange = sstable_info.key_range;
+                if keyrange.inf_key_range() {
+                    // For empty key range, we don't need to encode it
+                    // Timetravel will use the default key range to stripped the PbSstableInfo
+                    // Note: If new fields are added, using Default to implement stripped may not work, resulting in an increase in encode size.
+                    None
+                } else {
+                    let pb_key_range = PbKeyRange {
+                        left: keyrange.left.into(),
+                        right: keyrange.right.into(),
+                        right_exclusive: keyrange.right_exclusive,
+                    };
+                    Some(pb_key_range)
+                }
+            },
+
+            file_size: sstable_info.file_size,
+            table_ids: sstable_info.table_ids.clone(),
+            meta_offset: sstable_info.meta_offset,
+            stale_key_count: sstable_info.stale_key_count,
+            total_key_count: sstable_info.total_key_count,
+            min_epoch: sstable_info.min_epoch,
+            max_epoch: sstable_info.max_epoch,
+            uncompressed_file_size: sstable_info.uncompressed_file_size,
+            range_tombstone_count: sstable_info.range_tombstone_count,
+            bloom_filter_kind: sstable_info.bloom_filter_kind.into(),
+        }
+    }
+}
+
+impl From<&SstableInfo> for PbSstableInfo {
+    fn from(sstable_info: &SstableInfo) -> Self {
+        PbSstableInfo {
+            object_id: sstable_info.object_id,
+            sst_id: sstable_info.sst_id,
+            key_range: {
+                let keyrange = &sstable_info.key_range;
+                if keyrange.inf_key_range() {
+                    None
+                } else {
+                    let pb_key_range = PbKeyRange {
+                        left: keyrange.left.to_vec(),
+                        right: keyrange.right.to_vec(),
+                        right_exclusive: keyrange.right_exclusive,
+                    };
+                    Some(pb_key_range)
+                }
+            },
+
+            file_size: sstable_info.file_size,
+            table_ids: sstable_info.table_ids.clone(),
+            meta_offset: sstable_info.meta_offset,
+            stale_key_count: sstable_info.stale_key_count,
+            total_key_count: sstable_info.total_key_count,
+            min_epoch: sstable_info.min_epoch,
+            max_epoch: sstable_info.max_epoch,
+            uncompressed_file_size: sstable_info.uncompressed_file_size,
+            range_tombstone_count: sstable_info.range_tombstone_count,
+            bloom_filter_kind: sstable_info.bloom_filter_kind.into(),
+        }
+    }
+}
+
+impl SstableInfo {
+    pub fn remove_key_range(&mut self) {
+        self.key_range = KeyRange::default();
+    }
+}
diff --git a/src/storage/hummock_sdk/src/table_watermark.rs b/src/storage/hummock_sdk/src/table_watermark.rs
index 73f7bac358e1b..2de3fd1b69427 100644
--- a/src/storage/hummock_sdk/src/table_watermark.rs
+++ b/src/storage/hummock_sdk/src/table_watermark.rs
@@ -21,7 +21,7 @@ use std::sync::Arc;
 
 use bytes::Bytes;
 use itertools::Itertools;
-use risingwave_common::buffer::{Bitmap, BitmapBuilder};
+use risingwave_common::bitmap::{Bitmap, BitmapBuilder};
 use risingwave_common::catalog::TableId;
 use risingwave_common::hash::{VirtualNode, VnodeBitmapExt};
 use risingwave_common_estimate_size::EstimateSize;
@@ -45,16 +45,27 @@ pub struct TableWatermarksIndex {
     pub staging_watermarks: VecDeque<(HummockEpoch, Arc<[VnodeWatermark]>)>,
     pub committed_watermarks: Option<Arc<TableWatermarks>>,
     latest_epoch: HummockEpoch,
-    committed_epoch: HummockEpoch,
+    committed_epoch: Option<HummockEpoch>,
 }
 
 impl TableWatermarksIndex {
-    pub fn new(watermark_direction: WatermarkDirection, committed_epoch: HummockEpoch) -> Self {
+    pub fn new(
+        watermark_direction: WatermarkDirection,
+        first_epoch: HummockEpoch,
+        first_vnode_watermark: Vec<VnodeWatermark>,
+        committed_epoch: Option<HummockEpoch>,
+    ) -> Self {
+        if let Some(committed_epoch) = committed_epoch {
+            assert!(first_epoch > committed_epoch);
+        }
         Self {
             watermark_direction,
-            staging_watermarks: VecDeque::new(),
+            staging_watermarks: VecDeque::from_iter([(
+                first_epoch,
+                Arc::from(first_vnode_watermark),
+            )]),
             committed_watermarks: None,
-            latest_epoch: committed_epoch,
+            latest_epoch: first_epoch,
             committed_epoch,
         }
     }
@@ -66,7 +77,7 @@ impl TableWatermarksIndex {
         Self {
             watermark_direction: committed_watermarks.direction,
             staging_watermarks: VecDeque::new(),
-            committed_epoch,
+            committed_epoch: Some(committed_epoch),
             latest_epoch: committed_epoch,
             committed_watermarks: Some(committed_watermarks),
         }
@@ -238,11 +249,20 @@ impl TableWatermarksIndex {
         committed_epoch: HummockEpoch,
     ) {
         assert_eq!(self.watermark_direction, committed_watermark.direction);
-        assert!(self.committed_epoch <= committed_epoch);
-        if self.committed_epoch == committed_epoch {
-            return;
+        if let Some(prev_committed_epoch) = self.committed_epoch {
+            assert!(prev_committed_epoch <= committed_epoch);
+            if prev_committed_epoch == committed_epoch {
+                return;
+            }
+        }
+        if self.latest_epoch < committed_epoch {
+            warn!(
+                latest_epoch = self.latest_epoch,
+                committed_epoch, "committed_epoch exceed table watermark latest_epoch"
+            );
+            self.latest_epoch = committed_epoch;
         }
-        self.committed_epoch = committed_epoch;
+        self.committed_epoch = Some(committed_epoch);
         self.committed_watermarks = Some(committed_watermark);
         // keep only watermark higher than committed epoch
         while let Some((old_epoch, _)) = self.staging_watermarks.front()
@@ -300,26 +320,52 @@ impl VnodeWatermark {
         }
     }
 
+    pub fn vnode_bitmap(&self) -> &Bitmap {
+        &self.vnode_bitmap
+    }
+
+    pub fn watermark(&self) -> &Bytes {
+        &self.watermark
+    }
+
     pub fn to_protobuf(&self) -> PbVnodeWatermark {
-        PbVnodeWatermark {
-            watermark: self.watermark.to_vec(),
-            vnode_bitmap: Some(self.vnode_bitmap.to_protobuf()),
+        self.into()
+    }
+}
+
+impl From<PbVnodeWatermark> for VnodeWatermark {
+    fn from(pb: PbVnodeWatermark) -> Self {
+        Self {
+            vnode_bitmap: Arc::new(Bitmap::from(pb.vnode_bitmap.as_ref().unwrap())),
+            watermark: Bytes::from(pb.watermark),
         }
     }
+}
 
-    pub fn from_protobuf(pb: &PbVnodeWatermark) -> Self {
+impl From<&PbVnodeWatermark> for VnodeWatermark {
+    fn from(pb: &PbVnodeWatermark) -> Self {
         Self {
             vnode_bitmap: Arc::new(Bitmap::from(pb.vnode_bitmap.as_ref().unwrap())),
             watermark: Bytes::from(pb.watermark.clone()),
         }
     }
+}
 
-    pub fn vnode_bitmap(&self) -> &Bitmap {
-        &self.vnode_bitmap
+impl From<VnodeWatermark> for PbVnodeWatermark {
+    fn from(watermark: VnodeWatermark) -> Self {
+        Self {
+            watermark: watermark.watermark.into(),
+            vnode_bitmap: Some(watermark.vnode_bitmap.to_protobuf()),
+        }
     }
+}
 
-    pub fn watermark(&self) -> &Bytes {
-        &self.watermark
+impl From<&VnodeWatermark> for PbVnodeWatermark {
+    fn from(watermark: &VnodeWatermark) -> Self {
+        Self {
+            watermark: watermark.watermark.to_vec(),
+            vnode_bitmap: Some(watermark.vnode_bitmap.to_protobuf()),
+        }
     }
 }
 
@@ -342,38 +388,6 @@ impl TableWatermarks {
         }
     }
 
-    pub fn to_protobuf(&self) -> PbTableWatermarks {
-        PbTableWatermarks {
-            epoch_watermarks: self
-                .watermarks
-                .iter()
-                .map(|(epoch, watermarks)| PbEpochNewWatermarks {
-                    watermarks: watermarks.iter().map(VnodeWatermark::to_protobuf).collect(),
-                    epoch: *epoch,
-                })
-                .collect(),
-            is_ascending: match self.direction {
-                WatermarkDirection::Ascending => true,
-                WatermarkDirection::Descending => false,
-            },
-        }
-    }
-
-    pub fn estimated_encode_len(&self) -> usize {
-        self.watermarks.len() * size_of::<HummockEpoch>()
-            + self
-                .watermarks
-                .iter()
-                .map(|(_, watermarks)| {
-                    watermarks
-                        .iter()
-                        .map(|watermark| watermark.estimated_size())
-                        .sum::<usize>()
-                })
-                .sum::<usize>()
-            + size_of::<bool>() // for direction
-    }
-
     pub fn add_new_epoch_watermarks(
         &mut self,
         epoch: HummockEpoch,
@@ -397,7 +411,7 @@ impl TableWatermarks {
                     let watermarks = epoch_watermark
                         .watermarks
                         .iter()
-                        .map(VnodeWatermark::from_protobuf)
+                        .map(VnodeWatermark::from)
                         .collect_vec();
                     (epoch, Arc::from(watermarks))
                 })
@@ -560,6 +574,115 @@ impl TableWatermarks {
     }
 }
 
+impl TableWatermarks {
+    pub fn estimated_encode_len(&self) -> usize {
+        self.watermarks.len() * size_of::<HummockEpoch>()
+            + self
+                .watermarks
+                .iter()
+                .map(|(_, watermarks)| {
+                    watermarks
+                        .iter()
+                        .map(|watermark| watermark.estimated_size())
+                        .sum::<usize>()
+                })
+                .sum::<usize>()
+            + size_of::<bool>() // for direction
+    }
+
+    pub fn to_protobuf(&self) -> PbTableWatermarks {
+        self.into()
+    }
+}
+
+impl From<&PbTableWatermarks> for TableWatermarks {
+    fn from(pb: &PbTableWatermarks) -> Self {
+        Self {
+            watermarks: pb
+                .epoch_watermarks
+                .iter()
+                .map(|epoch_watermark| {
+                    let epoch = epoch_watermark.epoch;
+                    let watermarks = epoch_watermark
+                        .watermarks
+                        .iter()
+                        .map(VnodeWatermark::from)
+                        .collect();
+                    (epoch, watermarks)
+                })
+                .collect(),
+            direction: if pb.is_ascending {
+                WatermarkDirection::Ascending
+            } else {
+                WatermarkDirection::Descending
+            },
+        }
+    }
+}
+
+impl From<&TableWatermarks> for PbTableWatermarks {
+    fn from(table_watermarks: &TableWatermarks) -> Self {
+        Self {
+            epoch_watermarks: table_watermarks
+                .watermarks
+                .iter()
+                .map(|(epoch, watermarks)| PbEpochNewWatermarks {
+                    watermarks: watermarks.iter().map(|wm| wm.into()).collect(),
+                    epoch: *epoch,
+                })
+                .collect(),
+            is_ascending: match table_watermarks.direction {
+                WatermarkDirection::Ascending => true,
+                WatermarkDirection::Descending => false,
+            },
+        }
+    }
+}
+
+impl From<PbTableWatermarks> for TableWatermarks {
+    fn from(pb: PbTableWatermarks) -> Self {
+        Self {
+            watermarks: pb
+                .epoch_watermarks
+                .into_iter()
+                .map(|epoch_watermark| {
+                    let epoch = epoch_watermark.epoch;
+                    let watermarks = epoch_watermark
+                        .watermarks
+                        .into_iter()
+                        .map(VnodeWatermark::from)
+                        .collect();
+                    (epoch, watermarks)
+                })
+                .collect(),
+            direction: if pb.is_ascending {
+                WatermarkDirection::Ascending
+            } else {
+                WatermarkDirection::Descending
+            },
+        }
+    }
+}
+
+impl From<TableWatermarks> for PbTableWatermarks {
+    fn from(table_watermarks: TableWatermarks) -> Self {
+        Self {
+            epoch_watermarks: table_watermarks
+                .watermarks
+                .into_iter()
+                .map(|(epoch, watermarks)| PbEpochNewWatermarks {
+                    watermarks: watermarks.iter().map(PbVnodeWatermark::from).collect(),
+                    epoch,
+                })
+                .collect(),
+            is_ascending: match table_watermarks.direction {
+                WatermarkDirection::Ascending => true,
+                WatermarkDirection::Descending => false,
+            },
+        }
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use std::collections::Bound::Included;
@@ -570,7 +693,7 @@ mod tests {
 
     use bytes::Bytes;
     use itertools::Itertools;
-    use risingwave_common::buffer::{Bitmap, BitmapBuilder};
+    use risingwave_common::bitmap::{Bitmap, BitmapBuilder};
     use risingwave_common::catalog::TableId;
     use risingwave_common::hash::VirtualNode;
     use risingwave_common::util::epoch::{test_epoch, EpochExt};
@@ -902,11 +1025,11 @@ mod tests {
         watermark2: Bytes,
         watermark3: Bytes,
     ) -> TableWatermarksIndex {
-        let mut index = TableWatermarksIndex::new(direction, COMMITTED_EPOCH);
-        index.add_epoch_watermark(
-            EPOCH1,
-            vec![VnodeWatermark::new(build_bitmap(0..4), watermark1.clone())].into(),
+        let mut index = TableWatermarksIndex::new(
             direction,
+            EPOCH1,
+            vec![VnodeWatermark::new(build_bitmap(0..4), watermark1.clone())],
+            Some(COMMITTED_EPOCH),
         );
         index.add_epoch_watermark(
             EPOCH2,
@@ -1057,7 +1180,7 @@ mod tests {
                 .clone(),
             EPOCH1,
         );
-        assert_eq!(EPOCH1, index.committed_epoch);
+        assert_eq!(EPOCH1, index.committed_epoch.unwrap());
         assert_eq!(EPOCH2, index.latest_epoch);
         for vnode in 0..VirtualNode::COUNT {
             let vnode = VirtualNode::from_index(vnode);
diff --git a/src/storage/hummock_sdk/src/time_travel.rs b/src/storage/hummock_sdk/src/time_travel.rs
new file mode 100644
index 0000000000000..a870d7d6d549a
--- /dev/null
+++ b/src/storage/hummock_sdk/src/time_travel.rs
@@ -0,0 +1,345 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::{HashMap, HashSet};
+use std::sync::Arc;
+
+use risingwave_common::catalog::TableId;
+use risingwave_pb::hummock::hummock_version_delta::PbGroupDeltas;
+use risingwave_pb::hummock::{PbHummockVersion, PbHummockVersionDelta, PbStateTableInfoDelta};
+
+use crate::change_log::{ChangeLogDelta, EpochNewChangeLog, TableChangeLog};
+use crate::level::{Level, Levels, OverlappingLevel};
+use crate::sstable_info::SstableInfo;
+use crate::table_watermark::TableWatermarks;
+use crate::version::{
+    GroupDelta, GroupDeltas, HummockVersion, HummockVersionDelta, HummockVersionStateTableInfo,
+    IntraLevelDelta,
+};
+use crate::{CompactionGroupId, HummockSstableId};
+
+/// [`IncompleteHummockVersion`] is incomplete because `SSTableInfo` only has the `sst_id` set in the following fields:
+/// - `PbLevels`
+/// - `TableChangeLog`
+#[derive(Debug, Clone, PartialEq)]
+pub struct IncompleteHummockVersion {
+    pub id: u64,
+    pub levels: HashMap<CompactionGroupId, Levels>,
+    pub max_committed_epoch: u64,
+    safe_epoch: u64,
+    pub table_watermarks: HashMap<TableId, Arc<TableWatermarks>>,
+    pub table_change_log: HashMap<TableId, TableChangeLog>,
+    pub state_table_info: HummockVersionStateTableInfo,
+}
+
+/// Clone from an `SstableInfo`, but only set the `sst_id` for the target, leaving other fields as default.
+/// The goal is to reduce the size of pb object generated afterward.
+fn stripped_sstable_info(origin: &SstableInfo) -> SstableInfo {
+    SstableInfo {
+        object_id: Default::default(),
+        sst_id: origin.sst_id,
+        key_range: Default::default(),
+        file_size: Default::default(),
+        table_ids: Default::default(),
+        meta_offset: Default::default(),
+        stale_key_count: Default::default(),
+        total_key_count: Default::default(),
+        min_epoch: Default::default(),
+        max_epoch: Default::default(),
+        uncompressed_file_size: Default::default(),
+        range_tombstone_count: Default::default(),
+        bloom_filter_kind: Default::default(),
+    }
+}
+
+fn stripped_epoch_new_change_log(origin: &EpochNewChangeLog) -> EpochNewChangeLog {
+    EpochNewChangeLog {
+        old_value: origin.old_value.iter().map(stripped_sstable_info).collect(),
+        new_value: origin.new_value.iter().map(stripped_sstable_info).collect(),
+        epochs: origin.epochs.clone(),
+    }
+}
+
+fn stripped_change_log_delta(origin: &ChangeLogDelta) -> ChangeLogDelta {
+    ChangeLogDelta {
+        new_log: origin.new_log.as_ref().map(stripped_epoch_new_change_log),
+        truncate_epoch: origin.truncate_epoch,
+    }
+}
+
+fn stripped_level(origin: &Level) -> Level {
+    Level {
+        level_idx: origin.level_idx,
+        level_type: origin.level_type,
+        table_infos: origin
+            .table_infos
+            .iter()
+            .map(stripped_sstable_info)
+            .collect(),
+        total_file_size: origin.total_file_size,
+        sub_level_id: origin.sub_level_id,
+        uncompressed_file_size: origin.uncompressed_file_size,
+        vnode_partition_count: origin.vnode_partition_count,
+    }
+}
+
+pub fn refill_version(
+    version: &mut HummockVersion,
+    sst_id_to_info: &HashMap<HummockSstableId, SstableInfo>,
+) {
+    for level in version.levels.values_mut().flat_map(|level| {
+        level
+            .l0
+            .sub_levels
+            .iter_mut()
+            .rev()
+            .chain(level.levels.iter_mut())
+    }) {
+        refill_level(level, sst_id_to_info);
+    }
+
+    for t in version.table_change_log.values_mut() {
+        refill_table_change_log(t, sst_id_to_info);
+    }
+}
+
+fn refill_level(level: &mut Level, sst_id_to_info: &HashMap<HummockSstableId, SstableInfo>) {
+    for s in &mut level.table_infos {
+        refill_sstable_info(s, sst_id_to_info);
+    }
+}
+
+fn refill_table_change_log(
+    table_change_log: &mut TableChangeLog,
+    sst_id_to_info: &HashMap<HummockSstableId, SstableInfo>,
+) {
+    for c in &mut table_change_log.0 {
+        for s in &mut c.old_value {
+            refill_sstable_info(s, sst_id_to_info);
+        }
+        for s in &mut c.new_value {
+            refill_sstable_info(s, sst_id_to_info);
+        }
+    }
+}
+
+/// Caller should ensure `sst_id_to_info` includes an entry corresponding to `sstable_info`.
+fn refill_sstable_info(
+    sstable_info: &mut SstableInfo,
+    sst_id_to_info: &HashMap<HummockSstableId, SstableInfo>,
+) {
+    *sstable_info = sst_id_to_info
+        .get(&sstable_info.sst_id)
+        .unwrap_or_else(|| panic!("SstableInfo should exist"))
+        .clone();
+}
+
+fn stripped_l0(origin: &OverlappingLevel) -> OverlappingLevel {
+    OverlappingLevel {
+        sub_levels: origin.sub_levels.iter().map(stripped_level).collect(),
+        total_file_size: origin.total_file_size,
+        uncompressed_file_size: origin.uncompressed_file_size,
+    }
+}
+
+#[allow(deprecated)]
+fn stripped_levels(origin: &Levels) -> Levels {
+    Levels {
+        levels: origin.levels.iter().map(stripped_level).collect(),
+        l0: stripped_l0(&origin.l0),
+        group_id: origin.group_id,
+        parent_group_id: origin.parent_group_id,
+        member_table_ids: Default::default(),
+    }
+}
+
+fn stripped_intra_level_delta(origin: &IntraLevelDelta) -> IntraLevelDelta {
+    IntraLevelDelta {
+        level_idx: origin.level_idx,
+        l0_sub_level_id: origin.l0_sub_level_id,
+        removed_table_ids: origin.removed_table_ids.clone(),
+        inserted_table_infos: origin
+            .inserted_table_infos
+            .iter()
+            .map(stripped_sstable_info)
+            .collect(),
+        vnode_partition_count: origin.vnode_partition_count,
+    }
+}
+
+fn stripped_group_delta(origin: &GroupDelta) -> GroupDelta {
+    match origin {
+        GroupDelta::IntraLevel(l) => GroupDelta::IntraLevel(stripped_intra_level_delta(l)),
+        _ => panic!("time travel expects DeltaType::IntraLevel only"),
+    }
+}
+
+fn stripped_group_deltas(origin: &GroupDeltas) -> GroupDeltas {
+    let group_deltas = origin
+        .group_deltas
+        .iter()
+        .map(stripped_group_delta)
+        .collect();
+    GroupDeltas { group_deltas }
+}
+
+/// `SStableInfo` will be stripped.
+impl From<(&HummockVersion, &HashSet<CompactionGroupId>)> for IncompleteHummockVersion {
+    fn from(p: (&HummockVersion, &HashSet<CompactionGroupId>)) -> Self {
+        let (version, select_group) = p;
+        Self {
+            id: version.id,
+            levels: version
+                .levels
+                .iter()
+                .filter_map(|(group_id, levels)| {
+                    if select_group.contains(group_id) {
+                        Some((*group_id as CompactionGroupId, stripped_levels(levels)))
+                    } else {
+                        None
+                    }
+                })
+                .collect(),
+            max_committed_epoch: version.max_committed_epoch,
+            safe_epoch: version.visible_table_safe_epoch(),
+            table_watermarks: version.table_watermarks.clone(),
+            // TODO: optimization: strip table change log
+            table_change_log: version
+                .table_change_log
+                .iter()
+                .map(|(table_id, change_log)| {
+                    let incomplete_table_change_log = change_log
+                        .0
+                        .iter()
+                        .map(stripped_epoch_new_change_log)
+                        .collect();
+                    (*table_id, TableChangeLog(incomplete_table_change_log))
+                })
+                .collect(),
+            state_table_info: version.state_table_info.clone(),
+        }
+    }
+}
+
+impl IncompleteHummockVersion {
+    /// Resulted `SStableInfo` is incompelte.
+    pub fn to_protobuf(&self) -> PbHummockVersion {
+        PbHummockVersion {
+            id: self.id,
+            levels: self
+                .levels
+                .iter()
+                .map(|(group_id, levels)| (*group_id as _, levels.to_protobuf()))
+                .collect(),
+            max_committed_epoch: self.max_committed_epoch,
+            safe_epoch: self.safe_epoch,
+            table_watermarks: self
+                .table_watermarks
+                .iter()
+                .map(|(table_id, watermark)| (table_id.table_id, watermark.to_protobuf()))
+                .collect(),
+            table_change_logs: self
+                .table_change_log
+                .iter()
+                .map(|(table_id, change_log)| (table_id.table_id, change_log.to_protobuf()))
+                .collect(),
+            state_table_info: self.state_table_info.to_protobuf(),
+        }
+    }
+}
+
+/// [`IncompleteHummockVersionDelta`] is incomplete because `SSTableInfo` only has the `sst_id` set in the following fields:
+/// - `PbGroupDeltas`
+/// - `ChangeLogDelta`
+#[derive(Debug, PartialEq, Clone)]
+pub struct IncompleteHummockVersionDelta {
+    pub id: u64,
+    pub prev_id: u64,
+    pub group_deltas: HashMap<CompactionGroupId, PbGroupDeltas>,
+    pub max_committed_epoch: u64,
+    pub safe_epoch: u64,
+    pub trivial_move: bool,
+    pub new_table_watermarks: HashMap<TableId, TableWatermarks>,
+    pub removed_table_ids: HashSet<TableId>,
+    pub change_log_delta: HashMap<TableId, ChangeLogDelta>,
+    pub state_table_info_delta: HashMap<TableId, PbStateTableInfoDelta>,
+}
+
+/// `SStableInfo` will be stripped.
+impl From<(&HummockVersionDelta, &HashSet<CompactionGroupId>)> for IncompleteHummockVersionDelta {
+    fn from(p: (&HummockVersionDelta, &HashSet<CompactionGroupId>)) -> Self {
+        let (delta, select_group) = p;
+        Self {
+            id: delta.id,
+            prev_id: delta.prev_id,
+            group_deltas: delta
+                .group_deltas
+                .iter()
+                .filter_map(|(cg_id, deltas)| {
+                    if select_group.contains(cg_id) {
+                        Some((*cg_id, stripped_group_deltas(deltas).to_protobuf()))
+                    } else {
+                        None
+                    }
+                })
+                .collect(),
+            max_committed_epoch: delta.max_committed_epoch,
+            safe_epoch: delta.visible_table_safe_epoch(),
+            trivial_move: delta.trivial_move,
+            new_table_watermarks: delta.new_table_watermarks.clone(),
+            removed_table_ids: delta.removed_table_ids.clone(),
+            // TODO: optimization: strip table change log
+            change_log_delta: delta
+                .change_log_delta
+                .iter()
+                .map(|(table_id, log_delta)| (*table_id, stripped_change_log_delta(log_delta)))
+                .collect(),
+            state_table_info_delta: delta.state_table_info_delta.clone(),
+        }
+    }
+}
+
+impl IncompleteHummockVersionDelta {
+    /// Resulted `SStableInfo` is incompelte.
+    pub fn to_protobuf(&self) -> PbHummockVersionDelta {
+        PbHummockVersionDelta {
+            id: self.id,
+            prev_id: self.prev_id,
+            group_deltas: self.group_deltas.clone(),
+            max_committed_epoch: self.max_committed_epoch,
+            safe_epoch: self.safe_epoch,
+            trivial_move: self.trivial_move,
+            new_table_watermarks: self
+                .new_table_watermarks
+                .iter()
+                .map(|(table_id, watermarks)| (table_id.table_id, watermarks.to_protobuf()))
+                .collect(),
+            removed_table_ids: self
+                .removed_table_ids
+                .iter()
+                .map(|table_id| table_id.table_id)
+                .collect(),
+            change_log_delta: self
+                .change_log_delta
+                .iter()
+                .map(|(table_id, log_delta)| (table_id.table_id, log_delta.into()))
+                .collect(),
+            state_table_info_delta: self
+                .state_table_info_delta
+                .iter()
+                .map(|(table_id, delta)| (table_id.table_id, delta.clone()))
+                .collect(),
+        }
+    }
+}
diff --git a/src/storage/hummock_sdk/src/version.rs b/src/storage/hummock_sdk/src/version.rs
index 0315ee9b8f298..d38376a7fda0f 100644
--- a/src/storage/hummock_sdk/src/version.rs
+++ b/src/storage/hummock_sdk/src/version.rs
@@ -18,22 +18,23 @@ use std::mem::{replace, size_of};
 use std::ops::Deref;
 use std::sync::{Arc, LazyLock};
 
-use prost::Message;
+use itertools::Itertools;
 use risingwave_common::catalog::TableId;
 use risingwave_common::util::epoch::INVALID_EPOCH;
-use risingwave_pb::hummock::group_delta::DeltaType;
-use risingwave_pb::hummock::hummock_version::Levels as PbLevels;
-use risingwave_pb::hummock::hummock_version_delta::{ChangeLogDelta, GroupDeltas as PbGroupDeltas};
+use risingwave_pb::hummock::group_delta::PbDeltaType;
+use risingwave_pb::hummock::hummock_version_delta::PbGroupDeltas;
 use risingwave_pb::hummock::{
-    CompactionConfig, HummockVersion as PbHummockVersion,
-    HummockVersionDelta as PbHummockVersionDelta, SstableInfo, StateTableInfo as PbStateTableInfo,
-    StateTableInfo, StateTableInfoDelta,
+    CompactionConfig, PbGroupConstruct, PbGroupDelta, PbGroupDestroy, PbGroupMetaChange,
+    PbGroupTableChange, PbHummockVersion, PbHummockVersionDelta, PbIntraLevelDelta,
+    PbStateTableInfo, StateTableInfo, StateTableInfoDelta,
 };
 use tracing::warn;
 
-use crate::change_log::TableChangeLog;
+use crate::change_log::{ChangeLogDelta, TableChangeLog};
 use crate::compaction_group::hummock_version_ext::build_initial_compaction_group_levels;
 use crate::compaction_group::StaticCompactionGroupId;
+use crate::level::Levels;
+use crate::sstable_info::SstableInfo;
 use crate::table_watermark::TableWatermarks;
 use crate::{CompactionGroupId, HummockSstableObjectId, HummockVersionId, FIRST_VERSION_ID};
 
@@ -205,7 +206,7 @@ impl HummockVersionStateTableInfo {
 #[derive(Debug, Clone, PartialEq)]
 pub struct HummockVersion {
     pub id: u64,
-    pub levels: HashMap<CompactionGroupId, PbLevels>,
+    pub levels: HashMap<CompactionGroupId, Levels>,
     pub max_committed_epoch: u64,
     safe_epoch: u64,
     pub table_watermarks: HashMap<TableId, Arc<TableWatermarks>>,
@@ -215,7 +216,7 @@ pub struct HummockVersion {
 
 impl Default for HummockVersion {
     fn default() -> Self {
-        HummockVersion::from_protobuf_inner(&PbHummockVersion::default())
+        HummockVersion::from(&PbHummockVersion::default())
     }
 }
 
@@ -223,22 +224,45 @@ impl HummockVersion {
     /// Convert the `PbHummockVersion` received from rpc to `HummockVersion`. No need to
     /// maintain backward compatibility.
     pub fn from_rpc_protobuf(pb_version: &PbHummockVersion) -> Self {
-        Self::from_protobuf_inner(pb_version)
+        HummockVersion::from(pb_version)
     }
 
     /// Convert the `PbHummockVersion` deserialized from persisted state to `HummockVersion`.
     /// We should maintain backward compatibility.
     pub fn from_persisted_protobuf(pb_version: &PbHummockVersion) -> Self {
-        Self::from_protobuf_inner(pb_version)
+        HummockVersion::from(pb_version)
     }
 
-    fn from_protobuf_inner(pb_version: &PbHummockVersion) -> Self {
+    pub fn to_protobuf(&self) -> PbHummockVersion {
+        self.into()
+    }
+}
+
+impl HummockVersion {
+    pub fn estimated_encode_len(&self) -> usize {
+        self.levels.len() * size_of::<CompactionGroupId>()
+            + self
+                .levels
+                .values()
+                .map(|level| level.estimated_encode_len())
+                .sum::<usize>()
+            + self.table_watermarks.len() * size_of::<u32>()
+            + self
+                .table_watermarks
+                .values()
+                .map(|table_watermark| table_watermark.estimated_encode_len())
+                .sum::<usize>()
+    }
+}
+
+impl From<&PbHummockVersion> for HummockVersion {
+    fn from(pb_version: &PbHummockVersion) -> Self {
         Self {
             id: pb_version.id,
             levels: pb_version
                 .levels
                 .iter()
-                .map(|(group_id, levels)| (*group_id as CompactionGroupId, levels.clone()))
+                .map(|(group_id, levels)| (*group_id as CompactionGroupId, Levels::from(levels)))
                 .collect(),
             max_committed_epoch: pb_version.max_committed_epoch,
             safe_epoch: pb_version.safe_epoch,
@@ -248,7 +272,7 @@ impl HummockVersion {
                 .map(|(table_id, table_watermark)| {
                     (
                         TableId::new(*table_id),
-                        Arc::new(TableWatermarks::from_protobuf(table_watermark)),
+                        Arc::new(TableWatermarks::from(table_watermark)),
                     )
                 })
                 .collect(),
@@ -267,46 +291,61 @@ impl HummockVersion {
             ),
         }
     }
+}
 
-    pub fn to_protobuf(&self) -> PbHummockVersion {
-        PbHummockVersion {
-            id: self.id,
-            levels: self
+impl From<&HummockVersion> for PbHummockVersion {
+    fn from(version: &HummockVersion) -> Self {
+        Self {
+            id: version.id,
+            levels: version
                 .levels
                 .iter()
-                .map(|(group_id, levels)| (*group_id as _, levels.clone()))
+                .map(|(group_id, levels)| (*group_id as _, levels.into()))
                 .collect(),
-            max_committed_epoch: self.max_committed_epoch,
-            safe_epoch: self.safe_epoch,
-            table_watermarks: self
+            max_committed_epoch: version.max_committed_epoch,
+            safe_epoch: version.safe_epoch,
+            table_watermarks: version
                 .table_watermarks
                 .iter()
-                .map(|(table_id, watermark)| (table_id.table_id, watermark.to_protobuf()))
+                .map(|(table_id, watermark)| (table_id.table_id, watermark.as_ref().into()))
                 .collect(),
-            table_change_logs: self
+            table_change_logs: version
                 .table_change_log
                 .iter()
                 .map(|(table_id, change_log)| (table_id.table_id, change_log.to_protobuf()))
                 .collect(),
-            state_table_info: self.state_table_info.to_protobuf(),
+            state_table_info: version.state_table_info.to_protobuf(),
         }
     }
+}
 
-    pub fn estimated_encode_len(&self) -> usize {
-        self.levels.len() * size_of::<CompactionGroupId>()
-            + self
+impl From<HummockVersion> for PbHummockVersion {
+    fn from(version: HummockVersion) -> Self {
+        Self {
+            id: version.id,
+            levels: version
                 .levels
-                .values()
-                .map(|level| level.encoded_len())
-                .sum::<usize>()
-            + self.table_watermarks.len() * size_of::<u32>()
-            + self
+                .into_iter()
+                .map(|(group_id, levels)| (group_id as _, levels.into()))
+                .collect(),
+            max_committed_epoch: version.max_committed_epoch,
+            safe_epoch: version.safe_epoch,
+            table_watermarks: version
                 .table_watermarks
-                .values()
-                .map(|table_watermark| table_watermark.estimated_encode_len())
-                .sum::<usize>()
+                .into_iter()
+                .map(|(table_id, watermark)| (table_id.table_id, watermark.as_ref().into()))
+                .collect(),
+            table_change_logs: version
+                .table_change_log
+                .into_iter()
+                .map(|(table_id, change_log)| (table_id.table_id, change_log.to_protobuf()))
+                .collect(),
+            state_table_info: version.state_table_info.to_protobuf(),
+        }
     }
+}
 
+impl HummockVersion {
     pub fn next_version_id(&self) -> HummockVersionId {
         self.id + 1
     }
@@ -399,9 +438,9 @@ impl HummockVersion {
 pub struct HummockVersionDelta {
     pub id: u64,
     pub prev_id: u64,
-    pub group_deltas: HashMap<CompactionGroupId, PbGroupDeltas>,
+    pub group_deltas: HashMap<CompactionGroupId, GroupDeltas>,
     pub max_committed_epoch: u64,
-    pub safe_epoch: u64,
+    safe_epoch: u64,
     pub trivial_move: bool,
     pub new_table_watermarks: HashMap<TableId, TableWatermarks>,
     pub removed_table_ids: HashSet<TableId>,
@@ -411,7 +450,7 @@ pub struct HummockVersionDelta {
 
 impl Default for HummockVersionDelta {
     fn default() -> Self {
-        HummockVersionDelta::from_protobuf_inner(&PbHummockVersionDelta::default())
+        HummockVersionDelta::from(&PbHummockVersionDelta::default())
     }
 }
 
@@ -419,83 +458,199 @@ impl HummockVersionDelta {
     /// Convert the `PbHummockVersionDelta` deserialized from persisted state to `HummockVersionDelta`.
     /// We should maintain backward compatibility.
     pub fn from_persisted_protobuf(delta: &PbHummockVersionDelta) -> Self {
-        Self::from_protobuf_inner(delta)
+        Self::from(delta)
     }
 
     /// Convert the `PbHummockVersionDelta` received from rpc to `HummockVersionDelta`. No need to
     /// maintain backward compatibility.
     pub fn from_rpc_protobuf(delta: &PbHummockVersionDelta) -> Self {
-        Self::from_protobuf_inner(delta)
+        Self::from(delta)
+    }
+
+    pub fn to_protobuf(&self) -> PbHummockVersionDelta {
+        self.into()
+    }
+}
+
+impl HummockVersionDelta {
+    /// Get the newly added object ids from the version delta.
+    ///
+    /// Note: the result can be false positive because we only collect the set of sst object ids in the `inserted_table_infos`,
+    /// but it is possible that the object is moved or split from other compaction groups or levels.
+    pub fn newly_added_object_ids(&self) -> HashSet<HummockSstableObjectId> {
+        self.group_deltas
+            .values()
+            .flat_map(|group_deltas| {
+                group_deltas.group_deltas.iter().flat_map(|group_delta| {
+                    static EMPTY_VEC: Vec<SstableInfo> = Vec::new();
+                    let sst_slice = match group_delta {
+                        GroupDelta::IntraLevel(level_delta) => &level_delta.inserted_table_infos,
+                        GroupDelta::GroupConstruct(_)
+                        | GroupDelta::GroupDestroy(_)
+                        | GroupDelta::GroupMetaChange(_)
+                        | GroupDelta::GroupTableChange(_) => &EMPTY_VEC,
+                    };
+                    sst_slice.iter().map(|sst| sst.object_id)
+                })
+            })
+            .chain(self.change_log_delta.values().flat_map(|delta| {
+                let new_log = delta.new_log.as_ref().unwrap();
+                new_log
+                    .new_value
+                    .iter()
+                    .map(|sst| sst.object_id)
+                    .chain(new_log.old_value.iter().map(|sst| sst.object_id))
+            }))
+            .collect()
+    }
+
+    pub fn newly_added_sst_ids(&self) -> HashSet<HummockSstableObjectId> {
+        let ssts_from_group_deltas = self.group_deltas.values().flat_map(|group_deltas| {
+            group_deltas.group_deltas.iter().flat_map(|group_delta| {
+                static EMPTY_VEC: Vec<SstableInfo> = Vec::new();
+                let sst_slice = match group_delta {
+                    GroupDelta::IntraLevel(level_delta) => &level_delta.inserted_table_infos,
+                    GroupDelta::GroupConstruct(_)
+                    | GroupDelta::GroupDestroy(_)
+                    | GroupDelta::GroupMetaChange(_)
+                    | GroupDelta::GroupTableChange(_) => &EMPTY_VEC,
+                };
+                sst_slice.iter()
+            })
+        });
+
+        let ssts_from_change_log = self.change_log_delta.values().flat_map(|delta| {
+            let new_log = delta.new_log.as_ref().unwrap();
+            new_log.new_value.iter().chain(new_log.old_value.iter())
+        });
+
+        ssts_from_group_deltas
+            .chain(ssts_from_change_log)
+            .map(|sst| sst.object_id)
+            .collect()
+    }
+
+    pub fn newly_added_sst_infos<'a>(
+        &'a self,
+        select_group: &'a HashSet<CompactionGroupId>,
+    ) -> impl Iterator<Item = &SstableInfo> + 'a {
+        self.group_deltas
+            .iter()
+            .filter_map(|(cg_id, group_deltas)| {
+                if select_group.contains(cg_id) {
+                    Some(group_deltas)
+                } else {
+                    None
+                }
+            })
+            .flat_map(|group_deltas| {
+                group_deltas.group_deltas.iter().flat_map(|group_delta| {
+                    static EMPTY_VEC: Vec<SstableInfo> = Vec::new();
+                    let sst_slice = match group_delta {
+                        GroupDelta::IntraLevel(level_delta) => &level_delta.inserted_table_infos,
+                        GroupDelta::GroupConstruct(_)
+                        | GroupDelta::GroupDestroy(_)
+                        | GroupDelta::GroupMetaChange(_)
+                        | GroupDelta::GroupTableChange(_) => &EMPTY_VEC,
+                    };
+                    sst_slice.iter()
+                })
+            })
+            .chain(self.change_log_delta.values().flat_map(|delta| {
+                // TODO: optimization: strip table change log
+                let new_log = delta.new_log.as_ref().unwrap();
+                new_log.new_value.iter().chain(new_log.old_value.iter())
+            }))
+    }
+
+    pub fn visible_table_safe_epoch(&self) -> u64 {
+        self.safe_epoch
+    }
+
+    pub fn set_safe_epoch(&mut self, safe_epoch: u64) {
+        self.safe_epoch = safe_epoch;
     }
+}
 
-    fn from_protobuf_inner(delta: &PbHummockVersionDelta) -> Self {
+impl From<&PbHummockVersionDelta> for HummockVersionDelta {
+    fn from(pb_version_delta: &PbHummockVersionDelta) -> Self {
         Self {
-            id: delta.id,
-            prev_id: delta.prev_id,
-            group_deltas: delta.group_deltas.clone(),
-            max_committed_epoch: delta.max_committed_epoch,
-            safe_epoch: delta.safe_epoch,
-            trivial_move: delta.trivial_move,
-            new_table_watermarks: delta
+            id: pb_version_delta.id,
+            prev_id: pb_version_delta.prev_id,
+            group_deltas: pb_version_delta
+                .group_deltas
+                .iter()
+                .map(|(group_id, deltas)| {
+                    (*group_id as CompactionGroupId, GroupDeltas::from(deltas))
+                })
+                .collect(),
+            max_committed_epoch: pb_version_delta.max_committed_epoch,
+            safe_epoch: pb_version_delta.safe_epoch,
+            trivial_move: pb_version_delta.trivial_move,
+            new_table_watermarks: pb_version_delta
                 .new_table_watermarks
                 .iter()
                 .map(|(table_id, watermarks)| {
-                    (
-                        TableId::new(*table_id),
-                        TableWatermarks::from_protobuf(watermarks),
-                    )
+                    (TableId::new(*table_id), TableWatermarks::from(watermarks))
                 })
                 .collect(),
-            removed_table_ids: delta
+            removed_table_ids: pb_version_delta
                 .removed_table_ids
                 .iter()
                 .map(|table_id| TableId::new(*table_id))
                 .collect(),
-            change_log_delta: delta
+            change_log_delta: pb_version_delta
                 .change_log_delta
                 .iter()
                 .map(|(table_id, log_delta)| {
                     (
                         TableId::new(*table_id),
                         ChangeLogDelta {
-                            new_log: log_delta.new_log.clone(),
+                            new_log: log_delta.new_log.clone().map(Into::into),
                             truncate_epoch: log_delta.truncate_epoch,
                         },
                     )
                 })
                 .collect(),
-            state_table_info_delta: delta
+
+            state_table_info_delta: pb_version_delta
                 .state_table_info_delta
                 .iter()
                 .map(|(table_id, delta)| (TableId::new(*table_id), delta.clone()))
                 .collect(),
         }
     }
+}
 
-    pub fn to_protobuf(&self) -> PbHummockVersionDelta {
-        PbHummockVersionDelta {
-            id: self.id,
-            prev_id: self.prev_id,
-            group_deltas: self.group_deltas.clone(),
-            max_committed_epoch: self.max_committed_epoch,
-            safe_epoch: self.safe_epoch,
-            trivial_move: self.trivial_move,
-            new_table_watermarks: self
+impl From<&HummockVersionDelta> for PbHummockVersionDelta {
+    fn from(version_delta: &HummockVersionDelta) -> Self {
+        Self {
+            id: version_delta.id,
+            prev_id: version_delta.prev_id,
+            group_deltas: version_delta
+                .group_deltas
+                .iter()
+                .map(|(group_id, deltas)| (*group_id as _, deltas.into()))
+                .collect(),
+            max_committed_epoch: version_delta.max_committed_epoch,
+            safe_epoch: version_delta.safe_epoch,
+            trivial_move: version_delta.trivial_move,
+            new_table_watermarks: version_delta
                 .new_table_watermarks
                 .iter()
-                .map(|(table_id, watermarks)| (table_id.table_id, watermarks.to_protobuf()))
+                .map(|(table_id, watermarks)| (table_id.table_id, watermarks.into()))
                 .collect(),
-            removed_table_ids: self
+            removed_table_ids: version_delta
                 .removed_table_ids
                 .iter()
                 .map(|table_id| table_id.table_id)
                 .collect(),
-            change_log_delta: self
+            change_log_delta: version_delta
                 .change_log_delta
                 .iter()
-                .map(|(table_id, log_delta)| (table_id.table_id, log_delta.clone()))
+                .map(|(table_id, log_delta)| (table_id.table_id, log_delta.into()))
                 .collect(),
-            state_table_info_delta: self
+            state_table_info_delta: version_delta
                 .state_table_info_delta
                 .iter()
                 .map(|(table_id, delta)| (table_id.table_id, delta.clone()))
@@ -504,37 +659,349 @@ impl HummockVersionDelta {
     }
 }
 
-impl HummockVersionDelta {
-    /// Get the newly added object ids from the version delta.
-    ///
-    /// Note: the result can be false positive because we only collect the set of sst object ids in the `inserted_table_infos`,
-    /// but it is possible that the object is moved or split from other compaction groups or levels.
-    pub fn newly_added_object_ids(&self) -> HashSet<HummockSstableObjectId> {
-        self.group_deltas
-            .values()
-            .flat_map(|group_deltas| {
-                group_deltas.group_deltas.iter().flat_map(|group_delta| {
-                    group_delta.delta_type.iter().flat_map(|delta_type| {
-                        static EMPTY_VEC: Vec<SstableInfo> = Vec::new();
-                        let sst_slice = match delta_type {
-                            DeltaType::IntraLevel(level_delta) => &level_delta.inserted_table_infos,
-                            DeltaType::GroupConstruct(_)
-                            | DeltaType::GroupDestroy(_)
-                            | DeltaType::GroupMetaChange(_)
-                            | DeltaType::GroupTableChange(_) => &EMPTY_VEC,
-                        };
-                        sst_slice.iter().map(|sst| sst.object_id)
-                    })
+impl From<HummockVersionDelta> for PbHummockVersionDelta {
+    fn from(version_delta: HummockVersionDelta) -> Self {
+        Self {
+            id: version_delta.id,
+            prev_id: version_delta.prev_id,
+            group_deltas: version_delta
+                .group_deltas
+                .into_iter()
+                .map(|(group_id, deltas)| (group_id as _, deltas.into()))
+                .collect(),
+            max_committed_epoch: version_delta.max_committed_epoch,
+            safe_epoch: version_delta.safe_epoch,
+            trivial_move: version_delta.trivial_move,
+            new_table_watermarks: version_delta
+                .new_table_watermarks
+                .into_iter()
+                .map(|(table_id, watermarks)| (table_id.table_id, watermarks.into()))
+                .collect(),
+            removed_table_ids: version_delta
+                .removed_table_ids
+                .into_iter()
+                .map(|table_id| table_id.table_id)
+                .collect(),
+            change_log_delta: version_delta
+                .change_log_delta
+                .into_iter()
+                .map(|(table_id, log_delta)| (table_id.table_id, log_delta.into()))
+                .collect(),
+            state_table_info_delta: version_delta
+                .state_table_info_delta
+                .into_iter()
+                .map(|(table_id, delta)| (table_id.table_id, delta.clone()))
+                .collect(),
+        }
+    }
+}
+
+impl From<PbHummockVersionDelta> for HummockVersionDelta {
+    fn from(pb_version_delta: PbHummockVersionDelta) -> Self {
+        Self {
+            id: pb_version_delta.id,
+            prev_id: pb_version_delta.prev_id,
+            group_deltas: pb_version_delta
+                .group_deltas
+                .into_iter()
+                .map(|(group_id, deltas)| (group_id as CompactionGroupId, deltas.into()))
+                .collect(),
+            max_committed_epoch: pb_version_delta.max_committed_epoch,
+            safe_epoch: pb_version_delta.safe_epoch,
+            trivial_move: pb_version_delta.trivial_move,
+            new_table_watermarks: pb_version_delta
+                .new_table_watermarks
+                .into_iter()
+                .map(|(table_id, watermarks)| (TableId::new(table_id), watermarks.into()))
+                .collect(),
+            removed_table_ids: pb_version_delta
+                .removed_table_ids
+                .into_iter()
+                .map(TableId::new)
+                .collect(),
+            change_log_delta: pb_version_delta
+                .change_log_delta
+                .iter()
+                .map(|(table_id, log_delta)| {
+                    (
+                        TableId::new(*table_id),
+                        ChangeLogDelta {
+                            new_log: log_delta.new_log.clone().map(Into::into),
+                            truncate_epoch: log_delta.truncate_epoch,
+                        },
+                    )
                 })
-            })
-            .chain(self.change_log_delta.values().flat_map(|delta| {
-                let new_log = delta.new_log.as_ref().unwrap();
-                new_log
-                    .new_value
-                    .iter()
-                    .map(|sst| sst.object_id)
-                    .chain(new_log.old_value.iter().map(|sst| sst.object_id))
-            }))
-            .collect()
+                .collect(),
+            state_table_info_delta: pb_version_delta
+                .state_table_info_delta
+                .iter()
+                .map(|(table_id, delta)| (TableId::new(*table_id), delta.clone()))
+                .collect(),
+        }
+    }
+}
+
+#[derive(Debug, PartialEq, Clone)]
+pub struct IntraLevelDelta {
+    pub level_idx: u32,
+    pub l0_sub_level_id: u64,
+    pub removed_table_ids: Vec<u64>,
+    pub inserted_table_infos: Vec<SstableInfo>,
+    pub vnode_partition_count: u32,
+}
+
+impl IntraLevelDelta {
+    pub fn estimated_encode_len(&self) -> usize {
+        size_of::<u32>()
+            + size_of::<u64>()
+            + self.removed_table_ids.len() * size_of::<u32>()
+            + self
+                .inserted_table_infos
+                .iter()
+                .map(|sst| sst.estimated_encode_len())
+                .sum::<usize>()
+            + size_of::<u32>()
+    }
+}
+
+impl From<PbIntraLevelDelta> for IntraLevelDelta {
+    fn from(pb_intra_level_delta: PbIntraLevelDelta) -> Self {
+        Self {
+            level_idx: pb_intra_level_delta.level_idx,
+            l0_sub_level_id: pb_intra_level_delta.l0_sub_level_id,
+            removed_table_ids: pb_intra_level_delta.removed_table_ids.clone(),
+            inserted_table_infos: pb_intra_level_delta
+                .inserted_table_infos
+                .into_iter()
+                .map(SstableInfo::from)
+                .collect_vec(),
+            vnode_partition_count: pb_intra_level_delta.vnode_partition_count,
+        }
+    }
+}
+
+impl From<IntraLevelDelta> for PbIntraLevelDelta {
+    fn from(intra_level_delta: IntraLevelDelta) -> Self {
+        Self {
+            level_idx: intra_level_delta.level_idx,
+            l0_sub_level_id: intra_level_delta.l0_sub_level_id,
+            removed_table_ids: intra_level_delta.removed_table_ids.clone(),
+            inserted_table_infos: intra_level_delta
+                .inserted_table_infos
+                .into_iter()
+                .map(|sst| sst.into())
+                .collect_vec(),
+            vnode_partition_count: intra_level_delta.vnode_partition_count,
+        }
+    }
+}
+
+impl From<&IntraLevelDelta> for PbIntraLevelDelta {
+    fn from(intra_level_delta: &IntraLevelDelta) -> Self {
+        Self {
+            level_idx: intra_level_delta.level_idx,
+            l0_sub_level_id: intra_level_delta.l0_sub_level_id,
+            removed_table_ids: intra_level_delta.removed_table_ids.clone(),
+            inserted_table_infos: intra_level_delta
+                .inserted_table_infos
+                .iter()
+                .map(|sst| sst.into())
+                .collect_vec(),
+            vnode_partition_count: intra_level_delta.vnode_partition_count,
+        }
+    }
+}
+
+impl From<&PbIntraLevelDelta> for IntraLevelDelta {
+    fn from(pb_intra_level_delta: &PbIntraLevelDelta) -> Self {
+        Self {
+            level_idx: pb_intra_level_delta.level_idx,
+            l0_sub_level_id: pb_intra_level_delta.l0_sub_level_id,
+            removed_table_ids: pb_intra_level_delta.removed_table_ids.clone(),
+            inserted_table_infos: pb_intra_level_delta
+                .inserted_table_infos
+                .iter()
+                .map(SstableInfo::from)
+                .collect_vec(),
+            vnode_partition_count: pb_intra_level_delta.vnode_partition_count,
+        }
+    }
+}
+
+impl IntraLevelDelta {
+    pub fn new(
+        level_idx: u32,
+        l0_sub_level_id: u64,
+        removed_table_ids: Vec<u64>,
+        inserted_table_infos: Vec<SstableInfo>,
+        vnode_partition_count: u32,
+    ) -> Self {
+        Self {
+            level_idx,
+            l0_sub_level_id,
+            removed_table_ids,
+            inserted_table_infos,
+            vnode_partition_count,
+        }
+    }
+}
+
+#[derive(Debug, PartialEq, Clone)]
+pub enum GroupDelta {
+    IntraLevel(IntraLevelDelta),
+    GroupConstruct(PbGroupConstruct),
+    GroupDestroy(PbGroupDestroy),
+    GroupMetaChange(PbGroupMetaChange),
+
+    #[allow(dead_code)]
+    GroupTableChange(PbGroupTableChange),
+}
+
+impl From<PbGroupDelta> for GroupDelta {
+    fn from(pb_group_delta: PbGroupDelta) -> Self {
+        match pb_group_delta.delta_type {
+            Some(PbDeltaType::IntraLevel(pb_intra_level_delta)) => {
+                GroupDelta::IntraLevel(IntraLevelDelta::from(pb_intra_level_delta))
+            }
+            Some(PbDeltaType::GroupConstruct(pb_group_construct)) => {
+                GroupDelta::GroupConstruct(pb_group_construct)
+            }
+            Some(PbDeltaType::GroupDestroy(pb_group_destroy)) => {
+                GroupDelta::GroupDestroy(pb_group_destroy)
+            }
+            Some(PbDeltaType::GroupMetaChange(pb_group_meta_change)) => {
+                GroupDelta::GroupMetaChange(pb_group_meta_change)
+            }
+            Some(PbDeltaType::GroupTableChange(pb_group_table_change)) => {
+                GroupDelta::GroupTableChange(pb_group_table_change)
+            }
+            None => panic!("delta_type is not set"),
+        }
+    }
+}
+
+impl From<GroupDelta> for PbGroupDelta {
+    fn from(group_delta: GroupDelta) -> Self {
+        match group_delta {
+            GroupDelta::IntraLevel(intra_level_delta) => PbGroupDelta {
+                delta_type: Some(PbDeltaType::IntraLevel(intra_level_delta.into())),
+            },
+            GroupDelta::GroupConstruct(pb_group_construct) => PbGroupDelta {
+                delta_type: Some(PbDeltaType::GroupConstruct(pb_group_construct)),
+            },
+            GroupDelta::GroupDestroy(pb_group_destroy) => PbGroupDelta {
+                delta_type: Some(PbDeltaType::GroupDestroy(pb_group_destroy)),
+            },
+            GroupDelta::GroupMetaChange(pb_group_meta_change) => PbGroupDelta {
+                delta_type: Some(PbDeltaType::GroupMetaChange(pb_group_meta_change)),
+            },
+            GroupDelta::GroupTableChange(pb_group_table_change) => PbGroupDelta {
+                delta_type: Some(PbDeltaType::GroupTableChange(pb_group_table_change)),
+            },
+        }
+    }
+}
+
+impl From<&GroupDelta> for PbGroupDelta {
+    fn from(group_delta: &GroupDelta) -> Self {
+        match group_delta {
+            GroupDelta::IntraLevel(intra_level_delta) => PbGroupDelta {
+                delta_type: Some(PbDeltaType::IntraLevel(intra_level_delta.into())),
+            },
+            GroupDelta::GroupConstruct(pb_group_construct) => PbGroupDelta {
+                delta_type: Some(PbDeltaType::GroupConstruct(pb_group_construct.clone())),
+            },
+            GroupDelta::GroupDestroy(pb_group_destroy) => PbGroupDelta {
+                delta_type: Some(PbDeltaType::GroupDestroy(pb_group_destroy.clone())),
+            },
+            GroupDelta::GroupMetaChange(pb_group_meta_change) => PbGroupDelta {
+                delta_type: Some(PbDeltaType::GroupMetaChange(pb_group_meta_change.clone())),
+            },
+            GroupDelta::GroupTableChange(pb_group_table_change) => PbGroupDelta {
+                delta_type: Some(PbDeltaType::GroupTableChange(pb_group_table_change.clone())),
+            },
+        }
+    }
+}
+
+impl From<&PbGroupDelta> for GroupDelta {
+    fn from(pb_group_delta: &PbGroupDelta) -> Self {
+        match &pb_group_delta.delta_type {
+            Some(PbDeltaType::IntraLevel(pb_intra_level_delta)) => {
+                GroupDelta::IntraLevel(IntraLevelDelta::from(pb_intra_level_delta))
+            }
+            Some(PbDeltaType::GroupConstruct(pb_group_construct)) => {
+                GroupDelta::GroupConstruct(pb_group_construct.clone())
+            }
+            Some(PbDeltaType::GroupDestroy(pb_group_destroy)) => {
+                GroupDelta::GroupDestroy(pb_group_destroy.clone())
+            }
+            Some(PbDeltaType::GroupMetaChange(pb_group_meta_change)) => {
+                GroupDelta::GroupMetaChange(pb_group_meta_change.clone())
+            }
+            Some(PbDeltaType::GroupTableChange(pb_group_table_change)) => {
+                GroupDelta::GroupTableChange(pb_group_table_change.clone())
+            }
+            None => panic!("delta_type is not set"),
+        }
+    }
+}
+
+#[derive(Debug, PartialEq, Clone, Default)]
+pub struct GroupDeltas {
+    pub group_deltas: Vec<GroupDelta>,
+}
+
+impl From<PbGroupDeltas> for GroupDeltas {
+    fn from(pb_group_deltas: PbGroupDeltas) -> Self {
+        Self {
+            group_deltas: pb_group_deltas
+                .group_deltas
+                .into_iter()
+                .map(GroupDelta::from)
+                .collect_vec(),
+        }
+    }
+}
+
+impl From<GroupDeltas> for PbGroupDeltas {
+    fn from(group_deltas: GroupDeltas) -> Self {
+        Self {
+            group_deltas: group_deltas
+                .group_deltas
+                .into_iter()
+                .map(|group_delta| group_delta.into())
+                .collect_vec(),
+        }
+    }
+}
+
+impl From<&GroupDeltas> for PbGroupDeltas {
+    fn from(group_deltas: &GroupDeltas) -> Self {
+        Self {
+            group_deltas: group_deltas
+                .group_deltas
+                .iter()
+                .map(|group_delta| group_delta.into())
+                .collect_vec(),
+        }
+    }
+}
+
+impl From<&PbGroupDeltas> for GroupDeltas {
+    fn from(pb_group_deltas: &PbGroupDeltas) -> Self {
+        Self {
+            group_deltas: pb_group_deltas
+                .group_deltas
+                .iter()
+                .map(GroupDelta::from)
+                .collect_vec(),
+        }
+    }
+}
+
+impl GroupDeltas {
+    pub fn to_protobuf(&self) -> PbGroupDeltas {
+        self.into()
     }
 }
diff --git a/src/storage/hummock_test/src/bin/replay/replay_impl.rs b/src/storage/hummock_test/src/bin/replay/replay_impl.rs
index d8a2a2f0c24bd..fe9e5874e328e 100644
--- a/src/storage/hummock_test/src/bin/replay/replay_impl.rs
+++ b/src/storage/hummock_test/src/bin/replay/replay_impl.rs
@@ -19,7 +19,7 @@ use futures::{Stream, StreamExt, TryStreamExt};
 use futures_async_stream::try_stream;
 use risingwave_common::catalog::TableId;
 use risingwave_common::util::addr::HostAddr;
-use risingwave_common_service::observer_manager::{Channel, NotificationClient, ObserverError};
+use risingwave_common_service::{Channel, NotificationClient, ObserverError};
 use risingwave_hummock_sdk::key::TableKey;
 use risingwave_hummock_sdk::{HummockReadEpoch, SyncResult};
 use risingwave_hummock_trace::{
diff --git a/src/storage/hummock_test/src/compactor_tests.rs b/src/storage/hummock_test/src/compactor_tests.rs
index 2e2f8123be958..38ef095969a1f 100644
--- a/src/storage/hummock_test/src/compactor_tests.rs
+++ b/src/storage/hummock_test/src/compactor_tests.rs
@@ -15,7 +15,7 @@
 #[cfg(test)]
 pub(crate) mod tests {
 
-    use std::collections::{BTreeMap, BTreeSet, VecDeque};
+    use std::collections::{BTreeMap, BTreeSet, HashSet, VecDeque};
     use std::ops::Bound;
     use std::sync::Arc;
 
@@ -23,22 +23,25 @@ pub(crate) mod tests {
     use foyer::CacheContext;
     use itertools::Itertools;
     use rand::{Rng, RngCore, SeedableRng};
-    use risingwave_common::buffer::BitmapBuilder;
+    use risingwave_common::bitmap::BitmapBuilder;
     use risingwave_common::catalog::TableId;
     use risingwave_common::constants::hummock::CompactionFilterFlag;
     use risingwave_common::hash::VirtualNode;
     use risingwave_common::util::epoch::{test_epoch, Epoch, EpochExt};
-    use risingwave_common_service::observer_manager::NotificationClient;
+    use risingwave_common_service::NotificationClient;
     use risingwave_hummock_sdk::can_concat;
+    use risingwave_hummock_sdk::compact_task::CompactTask;
     use risingwave_hummock_sdk::compaction_group::StaticCompactionGroupId;
     use risingwave_hummock_sdk::key::{
         next_key, prefix_slice_with_vnode, prefixed_range_with_vnode, FullKey, TableKey,
         TABLE_PREFIX_LEN,
     };
-    use risingwave_hummock_sdk::prost_key_range::KeyRangeExt;
+    use risingwave_hummock_sdk::key_range::KeyRange;
+    use risingwave_hummock_sdk::level::InputLevel;
+    use risingwave_hummock_sdk::sstable_info::SstableInfo;
     use risingwave_hummock_sdk::table_stats::to_prost_table_stats_map;
     use risingwave_hummock_sdk::table_watermark::{
-        ReadTableWatermark, VnodeWatermark, WatermarkDirection,
+        ReadTableWatermark, TableWatermarks, VnodeWatermark, WatermarkDirection,
     };
     use risingwave_hummock_sdk::version::HummockVersion;
     use risingwave_meta::hummock::compaction::compaction_config::CompactionConfigBuilder;
@@ -51,10 +54,7 @@ pub(crate) mod tests {
     };
     use risingwave_meta::hummock::{HummockManagerRef, MockHummockMetaClient};
     use risingwave_pb::common::{HostAddress, WorkerType};
-    use risingwave_pb::hummock::table_watermarks::PbEpochNewWatermarks;
-    use risingwave_pb::hummock::{
-        CompactTask, InputLevel, KeyRange, SstableInfo, TableOption, TableWatermarks,
-    };
+    use risingwave_pb::hummock::TableOption;
     use risingwave_pb::meta::add_worker_node_request::Property;
     use risingwave_rpc_client::HummockMetaClient;
     use risingwave_storage::filter_key_extractor::{
@@ -156,6 +156,9 @@ pub(crate) mod tests {
         value_size: usize,
         epochs: Vec<u64>,
     ) {
+        for epoch in &epochs {
+            storage.start_epoch(*epoch, HashSet::from_iter([Default::default()]));
+        }
         let mut local = storage
             .new_local(NewLocalOptions::for_test(TableId::default()))
             .await;
@@ -315,7 +318,7 @@ pub(crate) mod tests {
                 .report_compact_task_for_test(
                     result_task.task_id,
                     Some(compact_task),
-                    result_task.task_status(),
+                    result_task.task_status,
                     result_task.sorted_output_ssts,
                     Some(to_prost_table_stats_map(task_stats)),
                 )
@@ -342,8 +345,6 @@ pub(crate) mod tests {
             .chain(
                 group
                     .l0
-                    .as_ref()
-                    .unwrap()
                     .sub_levels
                     .iter()
                     .flat_map(|level| level.table_infos.clone()),
@@ -470,7 +471,7 @@ pub(crate) mod tests {
             hummock_manager_ref
                 .report_compact_task(
                     result_task.task_id,
-                    result_task.task_status(),
+                    result_task.task_status,
                     result_task.sorted_output_ssts,
                     Some(to_prost_table_stats_map(task_stats)),
                 )
@@ -534,17 +535,16 @@ pub(crate) mod tests {
         existing_table_id: u32,
         keys_per_epoch: usize,
     ) {
+        let table_id = existing_table_id.into();
         let kv_count: u16 = 128;
         let mut epoch = test_epoch(1);
-        let mut local = storage
-            .new_local(NewLocalOptions::for_test(existing_table_id.into()))
-            .await;
+        let mut local = storage.new_local(NewLocalOptions::for_test(table_id)).await;
+
+        storage.start_epoch(epoch, HashSet::from_iter([table_id]));
 
         // 1. add sstables
         let val = Bytes::from(b"0"[..].repeat(1 << 10)); // 1024 Byte value
         for idx in 0..kv_count {
-            epoch.inc_epoch();
-
             if idx == 0 {
                 local.init_for_test(epoch).await.unwrap();
             }
@@ -559,9 +559,11 @@ pub(crate) mod tests {
             }
             local.flush().await.unwrap();
             let next_epoch = epoch.next_epoch();
+            storage.start_epoch(next_epoch, HashSet::from_iter([table_id]));
             local.seal_current_epoch(next_epoch, SealCurrentEpochOptions::for_test());
 
             flush_and_commit(&hummock_meta_client, storage, epoch).await;
+            epoch.inc_epoch();
         }
     }
 
@@ -727,9 +729,10 @@ pub(crate) mod tests {
             .await;
 
         let vnode = VirtualNode::from_index(1);
+        global_storage.start_epoch(epoch, HashSet::from_iter([1.into(), 2.into()]));
         for index in 0..kv_count {
-            epoch.inc_epoch();
             let next_epoch = epoch.next_epoch();
+            global_storage.start_epoch(next_epoch, HashSet::from_iter([1.into(), 2.into()]));
             if index == 0 {
                 storage_1.init_for_test(epoch).await.unwrap();
                 storage_2.init_for_test(epoch).await.unwrap();
@@ -755,6 +758,7 @@ pub(crate) mod tests {
 
             let res = global_storage.seal_and_sync_epoch(epoch).await.unwrap();
             hummock_meta_client.commit_epoch(epoch, res).await.unwrap();
+            epoch.inc_epoch();
         }
 
         // Mimic dropping table
@@ -797,11 +801,10 @@ pub(crate) mod tests {
             filter_key_extractor_manager,
         )
         .await;
-
         hummock_manager_ref
             .report_compact_task(
                 result_task.task_id,
-                result_task.task_status(),
+                result_task.task_status,
                 result_task.sorted_output_ssts,
                 Some(to_prost_table_stats_map(task_stats)),
             )
@@ -838,7 +841,6 @@ pub(crate) mod tests {
             .unwrap();
         assert!(compact_task.is_none());
 
-        epoch.inc_epoch();
         // to update version for hummock_storage
         global_storage.wait_version(version).await;
 
@@ -921,12 +923,14 @@ pub(crate) mod tests {
         let vnode = VirtualNode::from_index(1);
         let mut epoch_set = BTreeSet::new();
 
+        storage.start_epoch(epoch, HashSet::from_iter([existing_table_id.into()]));
+
         let mut local = storage
             .new_local(NewLocalOptions::for_test(existing_table_id.into()))
             .await;
         for i in 0..kv_count {
-            epoch += millisec_interval_epoch;
             let next_epoch = epoch + millisec_interval_epoch;
+            storage.start_epoch(next_epoch, HashSet::from_iter([existing_table_id.into()]));
             if i == 0 {
                 local.init_for_test(epoch).await.unwrap();
             }
@@ -944,6 +948,7 @@ pub(crate) mod tests {
 
             let res = storage.seal_and_sync_epoch(epoch).await.unwrap();
             hummock_meta_client.commit_epoch(epoch, res).await.unwrap();
+            epoch += millisec_interval_epoch;
         }
 
         let manual_compcation_option = ManualCompactionOption {
@@ -969,7 +974,10 @@ pub(crate) mod tests {
                 retention_seconds: Some(retention_seconds_expire_second),
             },
         )]);
-        compact_task.current_epoch_time = epoch;
+        compact_task.current_epoch_time = hummock_manager_ref
+            .get_current_version()
+            .await
+            .max_committed_epoch;
 
         // assert compact_task
         assert_eq!(
@@ -995,7 +1003,7 @@ pub(crate) mod tests {
         hummock_manager_ref
             .report_compact_task(
                 result_task.task_id,
-                result_task.task_status(),
+                result_task.task_status,
                 result_task.sorted_output_ssts,
                 Some(to_prost_table_stats_map(task_stats)),
             )
@@ -1123,12 +1131,13 @@ pub(crate) mod tests {
         let mut local = storage
             .new_local(NewLocalOptions::for_test(existing_table_id.into()))
             .await;
+        storage.start_epoch(epoch, HashSet::from_iter([existing_table_id.into()]));
         for i in 0..kv_count {
-            epoch += millisec_interval_epoch;
             if i == 0 {
                 local.init_for_test(epoch).await.unwrap();
             }
             let next_epoch = epoch + millisec_interval_epoch;
+            storage.start_epoch(next_epoch, HashSet::from_iter([existing_table_id.into()]));
             epoch_set.insert(epoch);
 
             let ramdom_key = [key_prefix.as_ref(), &rand::thread_rng().gen::<[u8; 32]>()].concat();
@@ -1139,6 +1148,7 @@ pub(crate) mod tests {
             local.seal_current_epoch(next_epoch, SealCurrentEpochOptions::for_test());
             let res = storage.seal_and_sync_epoch(epoch).await.unwrap();
             hummock_meta_client.commit_epoch(epoch, res).await.unwrap();
+            epoch += millisec_interval_epoch;
         }
 
         let manual_compcation_option = ManualCompactionOption {
@@ -1166,7 +1176,10 @@ pub(crate) mod tests {
 
         let compaction_filter_flag = CompactionFilterFlag::STATE_CLEAN | CompactionFilterFlag::TTL;
         compact_task.compaction_filter_mask = compaction_filter_flag.bits();
-        compact_task.current_epoch_time = epoch;
+        compact_task.current_epoch_time = hummock_manager_ref
+            .get_current_version()
+            .await
+            .max_committed_epoch;
 
         // 3. compact
         let (_tx, rx) = tokio::sync::oneshot::channel();
@@ -1182,7 +1195,7 @@ pub(crate) mod tests {
         hummock_manager_ref
             .report_compact_task(
                 result_task.task_id,
-                result_task.task_status(),
+                result_task.task_status,
                 result_task.sorted_output_ssts,
                 Some(to_prost_table_stats_map(task_stats)),
             )
@@ -1353,7 +1366,7 @@ pub(crate) mod tests {
         hummock_manager_ref
             .report_compact_task(
                 result_task.task_id,
-                result_task.task_status(),
+                result_task.task_status,
                 result_task.sorted_output_ssts,
                 Some(to_prost_table_stats_map(task_stats)),
             )
@@ -1389,10 +1402,6 @@ pub(crate) mod tests {
             normal_tables.push(sstable_store.sstable(sst_info, &mut stats).await.unwrap());
         }
         assert!(fast_ret.iter().all(|f| f.file_size < capacity * 6 / 5));
-        println!(
-            "fast sstables file size: {:?}",
-            fast_ret.iter().map(|f| f.file_size).collect_vec(),
-        );
         assert!(can_concat(&ret));
         assert!(can_concat(&fast_ret));
         let read_options = Arc::new(SstableIteratorReadOptions::default());
@@ -1522,7 +1531,6 @@ pub(crate) mod tests {
                 sstable_store.clone(),
             )
             .await;
-            println!("generate ssts size: {}", sst.file_size);
             ssts.push(sst);
         }
         let select_file_count = ssts.len() / 2;
@@ -1531,12 +1539,12 @@ pub(crate) mod tests {
             input_ssts: vec![
                 InputLevel {
                     level_idx: 5,
-                    level_type: 1,
+                    level_type: risingwave_pb::hummock::LevelType::Nonoverlapping,
                     table_infos: ssts.drain(..select_file_count).collect_vec(),
                 },
                 InputLevel {
                     level_idx: 6,
-                    level_type: 1,
+                    level_type: risingwave_pb::hummock::LevelType::Nonoverlapping,
                     table_infos: ssts,
                 },
             ],
@@ -1753,12 +1761,12 @@ pub(crate) mod tests {
             input_ssts: vec![
                 InputLevel {
                     level_idx: 5,
-                    level_type: 1,
+                    level_type: risingwave_pb::hummock::LevelType::Nonoverlapping,
                     table_infos: sst_infos.drain(..1).collect_vec(),
                 },
                 InputLevel {
                     level_idx: 6,
-                    level_type: 1,
+                    level_type: risingwave_pb::hummock::LevelType::Nonoverlapping,
                     table_infos: sst_infos,
                 },
             ],
@@ -1869,10 +1877,6 @@ pub(crate) mod tests {
             max_sst_file_size = std::cmp::max(max_sst_file_size, sst_info.file_size);
             sst_infos.push(sst_info);
         }
-        println!(
-            "input data: {}",
-            sst_infos.iter().map(|sst| sst.file_size).sum::<u64>(),
-        );
 
         let target_file_size = max_sst_file_size / 4;
         let mut table_watermarks = BTreeMap::default();
@@ -1891,13 +1895,11 @@ pub(crate) mod tests {
         table_watermarks.insert(
             1,
             TableWatermarks {
-                epoch_watermarks: vec![PbEpochNewWatermarks {
-                    watermarks: vec![
-                        VnodeWatermark::new(bitmap.clone(), watermark_key.clone()).to_protobuf()
-                    ],
-                    epoch: test_epoch(500),
-                }],
-                is_ascending: true,
+                watermarks: vec![(
+                    test_epoch(500),
+                    vec![VnodeWatermark::new(bitmap.clone(), watermark_key.clone())].into(),
+                )],
+                direction: WatermarkDirection::Ascending,
             },
         );
 
@@ -1905,12 +1907,12 @@ pub(crate) mod tests {
             input_ssts: vec![
                 InputLevel {
                     level_idx: 5,
-                    level_type: 1,
+                    level_type: risingwave_pb::hummock::LevelType::Nonoverlapping,
                     table_infos: sst_infos.drain(..1).collect_vec(),
                 },
                 InputLevel {
                     level_idx: 6,
-                    level_type: 1,
+                    level_type: risingwave_pb::hummock::LevelType::Nonoverlapping,
                     table_infos: sst_infos,
                 },
             ],
@@ -1927,12 +1929,6 @@ pub(crate) mod tests {
             ..Default::default()
         };
         let (ret, fast_ret) = run_fast_and_normal_runner(compact_ctx.clone(), task).await;
-        println!(
-            "normal compact result data: {}, fast compact result data: {}",
-            ret.iter().map(|sst| sst.file_size).sum::<u64>(),
-            fast_ret.iter().map(|sst| sst.file_size).sum::<u64>(),
-        );
-        // check_compaction_result(compact_ctx.sstable_store, ret.clone(), fast_ret, target_file_size).await;
         let mut fast_tables = Vec::with_capacity(fast_ret.len());
         let mut normal_tables = Vec::with_capacity(ret.len());
         let mut stats = StoreLocalStatistic::default();
diff --git a/src/storage/hummock_test/src/hummock_read_version_tests.rs b/src/storage/hummock_test/src/hummock_read_version_tests.rs
index 2329970b485f5..67c3aa8b059bd 100644
--- a/src/storage/hummock_test/src/hummock_read_version_tests.rs
+++ b/src/storage/hummock_test/src/hummock_read_version_tests.rs
@@ -19,14 +19,15 @@ use std::sync::Arc;
 use bytes::Bytes;
 use itertools::Itertools;
 use parking_lot::RwLock;
-use risingwave_common::buffer::Bitmap;
+use risingwave_common::bitmap::Bitmap;
 use risingwave_common::catalog::TableId;
 use risingwave_common::hash::VirtualNode;
 use risingwave_common::util::epoch::{test_epoch, EpochExt};
 use risingwave_hummock_sdk::key::{key_with_epoch, map_table_key_range};
+use risingwave_hummock_sdk::key_range::KeyRange;
+use risingwave_hummock_sdk::sstable_info::SstableInfo;
 use risingwave_hummock_sdk::LocalSstableInfo;
 use risingwave_meta::hummock::test_utils::setup_compute_env;
-use risingwave_pb::hummock::{KeyRange, SstableInfo};
 use risingwave_storage::hummock::event_handler::TEST_LOCAL_INSTANCE_ID;
 use risingwave_storage::hummock::iterator::test_utils::{
     iterator_test_table_key_of, iterator_test_user_key_of,
@@ -153,11 +154,13 @@ async fn test_read_version_basic() {
                 LocalSstableInfo::for_test(SstableInfo {
                     object_id: 1,
                     sst_id: 1,
-                    key_range: Some(KeyRange {
-                        left: key_with_epoch(iterator_test_user_key_of(1).encode(), test_epoch(1)),
-                        right: key_with_epoch(iterator_test_user_key_of(2).encode(), test_epoch(2)),
+                    key_range: KeyRange {
+                        left: key_with_epoch(iterator_test_user_key_of(1).encode(), test_epoch(1))
+                            .into(),
+                        right: key_with_epoch(iterator_test_user_key_of(2).encode(), test_epoch(2))
+                            .into(),
                         right_exclusive: false,
-                    }),
+                    },
                     file_size: 1,
                     table_ids: vec![0],
                     meta_offset: 1,
@@ -169,11 +172,13 @@ async fn test_read_version_basic() {
                 LocalSstableInfo::for_test(SstableInfo {
                     object_id: 2,
                     sst_id: 2,
-                    key_range: Some(KeyRange {
-                        left: key_with_epoch(iterator_test_user_key_of(3).encode(), test_epoch(3)),
-                        right: key_with_epoch(iterator_test_user_key_of(3).encode(), test_epoch(3)),
+                    key_range: KeyRange {
+                        left: key_with_epoch(iterator_test_user_key_of(3).encode(), test_epoch(3))
+                            .into(),
+                        right: key_with_epoch(iterator_test_user_key_of(3).encode(), test_epoch(3))
+                            .into(),
                         right_exclusive: false,
-                    }),
+                    },
                     file_size: 1,
                     table_ids: vec![0],
                     meta_offset: 1,
@@ -233,8 +238,8 @@ async fn test_read_version_basic() {
 
         let staging_ssts = staging_sst_iter.cloned().collect_vec();
         assert_eq!(2, staging_ssts.len());
-        assert_eq!(1, staging_ssts[0].get_object_id());
-        assert_eq!(2, staging_ssts[1].get_object_id());
+        assert_eq!(1, staging_ssts[0].object_id);
+        assert_eq!(2, staging_ssts[1].object_id);
     }
 
     {
@@ -257,7 +262,7 @@ async fn test_read_version_basic() {
 
         let staging_ssts = staging_sst_iter.cloned().collect_vec();
         assert_eq!(1, staging_ssts.len());
-        assert_eq!(2, staging_ssts[0].get_object_id());
+        assert_eq!(2, staging_ssts[0].object_id);
     }
 }
 
diff --git a/src/storage/hummock_test/src/hummock_storage_tests.rs b/src/storage/hummock_test/src/hummock_storage_tests.rs
index 5c6e1607de926..b3e304305660c 100644
--- a/src/storage/hummock_test/src/hummock_storage_tests.rs
+++ b/src/storage/hummock_test/src/hummock_storage_tests.rs
@@ -1,3 +1,4 @@
+use std::collections::{BTreeMap, HashMap, HashSet};
 // Copyright 2024 RisingWave Labs
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
@@ -19,18 +20,21 @@ use bytes::{BufMut, Bytes};
 use foyer::CacheContext;
 use futures::TryStreamExt;
 use itertools::Itertools;
-use risingwave_common::buffer::BitmapBuilder;
+use risingwave_common::bitmap::BitmapBuilder;
 use risingwave_common::catalog::TableId;
 use risingwave_common::hash::VirtualNode;
 use risingwave_common::range::RangeBoundsExt;
 use risingwave_common::util::epoch::{test_epoch, EpochExt};
+use risingwave_hummock_sdk::compaction_group::StaticCompactionGroupId;
 use risingwave_hummock_sdk::key::{
     gen_key_from_bytes, prefixed_range_with_vnode, FullKey, TableKey, UserKey, TABLE_PREFIX_LEN,
 };
+use risingwave_hummock_sdk::sstable_info::SstableInfo;
 use risingwave_hummock_sdk::table_watermark::{
     TableWatermarksIndex, VnodeWatermark, WatermarkDirection,
 };
-use risingwave_hummock_sdk::EpochWithGap;
+use risingwave_hummock_sdk::{EpochWithGap, LocalSstableInfo};
+use risingwave_meta::hummock::{BatchCommitForNewCg, CommitEpochInfo};
 use risingwave_rpc_client::HummockMetaClient;
 use risingwave_storage::hummock::local_version::pinned_version::PinnedVersion;
 use risingwave_storage::hummock::store::version::read_filter_for_version;
@@ -101,6 +105,9 @@ async fn test_storage_basic() {
 
     // epoch 0 is reserved by storage service
     let epoch1 = test_epoch(1);
+    test_env
+        .storage
+        .start_epoch(epoch1, HashSet::from_iter([TEST_TABLE_ID]));
     hummock_storage.init_for_test(epoch1).await.unwrap();
 
     // Write the first batch.
@@ -165,6 +172,9 @@ async fn test_storage_basic() {
     assert_eq!(value, None);
 
     let epoch2 = epoch1.next_epoch();
+    test_env
+        .storage
+        .start_epoch(epoch2, HashSet::from_iter([TEST_TABLE_ID]));
     hummock_storage.seal_current_epoch(epoch2, SealCurrentEpochOptions::for_test());
     hummock_storage
         .ingest_batch(
@@ -197,6 +207,9 @@ async fn test_storage_basic() {
 
     // Write the third batch.
     let epoch3 = epoch2.next_epoch();
+    test_env
+        .storage
+        .start_epoch(epoch3, HashSet::from_iter([TEST_TABLE_ID]));
     hummock_storage.seal_current_epoch(epoch3, SealCurrentEpochOptions::for_test());
     hummock_storage
         .ingest_batch(
@@ -457,6 +470,9 @@ async fn test_state_store_sync() {
 
     let base_epoch = read_version.read().committed().max_committed_epoch();
     let epoch1 = test_epoch(base_epoch.next_epoch());
+    test_env
+        .storage
+        .start_epoch(epoch1, HashSet::from_iter([TEST_TABLE_ID]));
     hummock_storage.init_for_test(epoch1).await.unwrap();
 
     // ingest 16B batch
@@ -511,6 +527,9 @@ async fn test_state_store_sync() {
         .unwrap();
 
     let epoch2 = epoch1.next_epoch();
+    test_env
+        .storage
+        .start_epoch(epoch2, HashSet::from_iter([TEST_TABLE_ID]));
     hummock_storage.seal_current_epoch(epoch2, SealCurrentEpochOptions::for_test());
 
     // ingest more 8B then will trigger a sync behind the scene
@@ -531,6 +550,9 @@ async fn test_state_store_sync() {
         .unwrap();
 
     let epoch3 = epoch2.next_epoch();
+    test_env
+        .storage
+        .start_epoch(epoch3, HashSet::from_iter([TEST_TABLE_ID]));
     hummock_storage.seal_current_epoch(epoch3, SealCurrentEpochOptions::for_test());
 
     let res = test_env.storage.seal_and_sync_epoch(epoch1).await.unwrap();
@@ -809,6 +831,9 @@ async fn test_delete_get() {
         .max_committed_epoch();
 
     let epoch1 = initial_epoch.next_epoch();
+    test_env
+        .storage
+        .start_epoch(epoch1, HashSet::from_iter([TEST_TABLE_ID]));
 
     hummock_storage.init_for_test(epoch1).await.unwrap();
     let batch1 = vec![
@@ -833,6 +858,9 @@ async fn test_delete_get() {
         .unwrap();
 
     let epoch2 = epoch1.next_epoch();
+    test_env
+        .storage
+        .start_epoch(epoch2, HashSet::from_iter([TEST_TABLE_ID]));
     hummock_storage.seal_current_epoch(epoch2, SealCurrentEpochOptions::for_test());
     let res = test_env.storage.seal_and_sync_epoch(epoch1).await.unwrap();
     test_env
@@ -896,6 +924,9 @@ async fn test_multiple_epoch_sync() {
         .max_committed_epoch();
 
     let epoch1 = initial_epoch.next_epoch();
+    test_env
+        .storage
+        .start_epoch(epoch1, HashSet::from_iter([TEST_TABLE_ID]));
     hummock_storage.init_for_test(epoch1).await.unwrap();
     let batch1 = vec![
         (
@@ -919,6 +950,9 @@ async fn test_multiple_epoch_sync() {
         .unwrap();
 
     let epoch2 = epoch1.next_epoch();
+    test_env
+        .storage
+        .start_epoch(epoch2, HashSet::from_iter([TEST_TABLE_ID]));
     hummock_storage.seal_current_epoch(epoch2, SealCurrentEpochOptions::for_test());
     let batch2 = vec![(
         gen_key_from_str(VirtualNode::ZERO, "bb"),
@@ -936,6 +970,9 @@ async fn test_multiple_epoch_sync() {
         .unwrap();
 
     let epoch3 = epoch2.next_epoch();
+    test_env
+        .storage
+        .start_epoch(epoch3, HashSet::from_iter([TEST_TABLE_ID]));
     hummock_storage.seal_current_epoch(epoch3, SealCurrentEpochOptions::for_test());
     let batch3 = vec![
         (
@@ -1011,6 +1048,9 @@ async fn test_multiple_epoch_sync() {
     test_get().await;
 
     let epoch4 = epoch3.next_epoch();
+    test_env
+        .storage
+        .start_epoch(epoch4, HashSet::from_iter([TEST_TABLE_ID]));
     hummock_storage.seal_current_epoch(epoch4, SealCurrentEpochOptions::for_test());
     test_env.storage.seal_epoch(epoch1, false);
     let sync_result2 = test_env.storage.seal_and_sync_epoch(epoch2).await.unwrap();
@@ -1043,6 +1083,9 @@ async fn test_iter_with_min_epoch() {
         .await;
 
     let epoch1 = (31 * 1000) << 16;
+    test_env
+        .storage
+        .start_epoch(epoch1, HashSet::from_iter([TEST_TABLE_ID]));
 
     let gen_key = |index: usize| -> TableKey<Bytes> {
         gen_key_from_str(VirtualNode::ZERO, format!("\0\0key_{}", index).as_str())
@@ -1069,6 +1112,9 @@ async fn test_iter_with_min_epoch() {
         .unwrap();
 
     let epoch2 = (32 * 1000) << 16;
+    test_env
+        .storage
+        .start_epoch(epoch2, HashSet::from_iter([TEST_TABLE_ID]));
     hummock_storage.seal_current_epoch(epoch2, SealCurrentEpochOptions::for_test());
     // epoch 2 write
     let batch_epoch2: Vec<(TableKey<Bytes>, StorageValue)> = (20..30)
@@ -1087,6 +1133,9 @@ async fn test_iter_with_min_epoch() {
         .unwrap();
 
     let epoch3 = (33 * 1000) << 16;
+    test_env
+        .storage
+        .start_epoch(epoch3, HashSet::from_iter([TEST_TABLE_ID]));
     hummock_storage.seal_current_epoch(epoch3, SealCurrentEpochOptions::for_test());
 
     {
@@ -1279,6 +1328,9 @@ async fn test_hummock_version_reader() {
     let hummock_version_reader = test_env.storage.version_reader();
 
     let epoch1 = (31 * 1000) << 16;
+    test_env
+        .storage
+        .start_epoch(epoch1, HashSet::from_iter([TEST_TABLE_ID]));
 
     let gen_key = |index: usize| -> TableKey<Bytes> {
         gen_key_from_str(VirtualNode::ZERO, format!("\0\0key_{}", index).as_str())
@@ -1292,12 +1344,18 @@ async fn test_hummock_version_reader() {
         .collect();
 
     let epoch2 = (32 * 1000) << 16;
+    test_env
+        .storage
+        .start_epoch(epoch2, HashSet::from_iter([TEST_TABLE_ID]));
     // epoch 2 write
     let batch_epoch2: Vec<(TableKey<Bytes>, StorageValue)> = (20..30)
         .map(|index| (gen_key(index), StorageValue::new_put(gen_val(index))))
         .collect();
 
     let epoch3 = (33 * 1000) << 16;
+    test_env
+        .storage
+        .start_epoch(epoch3, HashSet::from_iter([TEST_TABLE_ID]));
     // epoch 3 write
     let batch_epoch3: Vec<(TableKey<Bytes>, StorageValue)> = (40..50)
         .map(|index| (gen_key(index), StorageValue::new_put(gen_val(index))))
@@ -1340,6 +1398,9 @@ async fn test_hummock_version_reader() {
             .unwrap();
 
         let epoch4 = (34 * 1000) << 16;
+        test_env
+            .storage
+            .start_epoch(epoch4, HashSet::from_iter([TEST_TABLE_ID]));
         hummock_storage.seal_current_epoch(epoch4, SealCurrentEpochOptions::for_test());
 
         {
@@ -1710,6 +1771,9 @@ async fn test_get_with_min_epoch() {
         .await;
 
     let epoch1 = (31 * 1000) << 16;
+    test_env
+        .storage
+        .start_epoch(epoch1, HashSet::from_iter([TEST_TABLE_ID]));
     hummock_storage.init_for_test(epoch1).await.unwrap();
 
     let gen_key = |index: usize| -> TableKey<Bytes> {
@@ -1735,6 +1799,9 @@ async fn test_get_with_min_epoch() {
         .unwrap();
 
     let epoch2 = (32 * 1000) << 16;
+    test_env
+        .storage
+        .start_epoch(epoch2, HashSet::from_iter([TEST_TABLE_ID]));
     hummock_storage.seal_current_epoch(epoch2, SealCurrentEpochOptions::for_test());
     // epoch 2 write
     let batch_epoch2: Vec<(TableKey<Bytes>, StorageValue)> = (20..30)
@@ -1969,6 +2036,9 @@ async fn test_table_watermark() {
     });
 
     let epoch1 = (31 * 1000) << 16;
+    test_env
+        .storage
+        .start_epoch(epoch1, HashSet::from_iter([TEST_TABLE_ID]));
     local1.init_for_test(epoch1).await.unwrap();
     local1.update_vnode_bitmap(vnode_bitmap1.clone());
     local2.init_for_test(epoch1).await.unwrap();
@@ -2057,6 +2127,9 @@ async fn test_table_watermark() {
     let watermark1 = 10;
 
     let epoch2 = (32 * 1000) << 16;
+    test_env
+        .storage
+        .start_epoch(epoch2, HashSet::from_iter([TEST_TABLE_ID]));
     for (local, vnode_bitmap) in [
         (&mut local1, vnode_bitmap1.clone()),
         (&mut local2, vnode_bitmap2.clone()),
@@ -2159,6 +2232,9 @@ async fn test_table_watermark() {
     let batch2_epoch2 = gen_batch(vnode2, epoch2_indexes());
 
     let epoch3 = (33 * 1000) << 16;
+    test_env
+        .storage
+        .start_epoch(epoch3, HashSet::from_iter([TEST_TABLE_ID]));
 
     for (local, batch) in [(&mut local1, batch1_epoch2), (&mut local2, batch2_epoch2)] {
         for (key, value) in batch {
@@ -2372,6 +2448,9 @@ async fn test_table_watermark() {
     let (mut local1, mut local2) = test_after_epoch2(local1, local2).await;
 
     let epoch4 = (34 * 1000) << 16;
+    test_env
+        .storage
+        .start_epoch(epoch4, HashSet::from_iter([TEST_TABLE_ID]));
 
     for (local, vnode_bitmap) in [
         (&mut local1, vnode_bitmap1.clone()),
@@ -2403,3 +2482,151 @@ async fn test_table_watermark() {
 
     test_global_read(test_env.storage.clone(), epoch3).await;
 }
+
+#[tokio::test]
+async fn test_commit_multi_epoch() {
+    let test_env = prepare_hummock_test_env().await;
+    let context_id = test_env.meta_client.context_id();
+    let context_id_map = |object_ids: &[_]| {
+        HashMap::from_iter(object_ids.iter().map(|object_id| (*object_id, context_id)))
+    };
+    let existing_table_id = TableId::new(1);
+    test_env.register_table_id(existing_table_id).await;
+    let initial_epoch = test_env
+        .manager
+        .get_current_version()
+        .await
+        .max_committed_epoch;
+
+    let epoch1 = initial_epoch.next_epoch();
+    let sst1_epoch1 = SstableInfo {
+        sst_id: 11,
+        object_id: 1,
+        table_ids: vec![existing_table_id.table_id],
+        ..Default::default()
+    };
+    test_env
+        .manager
+        .commit_epoch(CommitEpochInfo {
+            sstables: vec![LocalSstableInfo {
+                sst_info: sst1_epoch1.clone(),
+                table_stats: Default::default(),
+            }],
+            new_table_watermarks: Default::default(),
+            sst_to_context: context_id_map(&[sst1_epoch1.object_id]),
+            new_table_fragment_info: None,
+            change_log_delta: Default::default(),
+            table_committed_epoch: BTreeMap::from_iter([(
+                epoch1,
+                HashSet::from_iter([existing_table_id]),
+            )]),
+            max_committed_epoch: epoch1,
+            batch_commit_for_new_cg: vec![],
+        })
+        .await
+        .unwrap();
+
+    let old_cg_id_set: HashSet<_> = {
+        let old_version = test_env.manager.get_current_version().await;
+        let cg = old_version
+            .levels
+            .get(&(StaticCompactionGroupId::StateDefault as _))
+            .unwrap();
+        let sub_levels = &cg.l0.sub_levels;
+        assert_eq!(sub_levels.len(), 1);
+        let sub_level = &sub_levels[0];
+        assert_eq!(sub_level.sub_level_id, epoch1);
+        assert_eq!(sub_level.table_infos.len(), 1);
+        assert_eq!(sub_level.table_infos[0].object_id, sst1_epoch1.object_id);
+        old_version.levels.keys().cloned().collect()
+    };
+
+    let epoch2 = epoch1.next_epoch();
+
+    let batch_commit_table_id = TableId::new(2);
+
+    let sst_epoch2 = SstableInfo {
+        sst_id: 22,
+        object_id: 2,
+        table_ids: vec![existing_table_id.table_id, batch_commit_table_id.table_id],
+        ..Default::default()
+    };
+
+    let sst2_epoch1 = SstableInfo {
+        sst_id: 33,
+        object_id: 3,
+        table_ids: vec![batch_commit_table_id.table_id],
+        ..Default::default()
+    };
+
+    test_env
+        .manager
+        .commit_epoch(CommitEpochInfo {
+            sstables: vec![LocalSstableInfo {
+                sst_info: sst_epoch2.clone(),
+                table_stats: Default::default(),
+            }],
+            new_table_watermarks: Default::default(),
+            sst_to_context: context_id_map(&[sst_epoch2.object_id, sst2_epoch1.object_id]),
+            new_table_fragment_info: None,
+            change_log_delta: Default::default(),
+            table_committed_epoch: BTreeMap::from_iter([(
+                epoch2,
+                HashSet::from_iter([existing_table_id, batch_commit_table_id]),
+            )]),
+            max_committed_epoch: epoch2,
+            batch_commit_for_new_cg: vec![BatchCommitForNewCg {
+                epoch_to_ssts: BTreeMap::from_iter([(
+                    epoch1,
+                    vec![LocalSstableInfo {
+                        sst_info: sst2_epoch1.clone(),
+                        table_stats: Default::default(),
+                    }],
+                )]),
+                table_ids: vec![batch_commit_table_id],
+            }],
+        })
+        .await
+        .unwrap();
+
+    let new_version = test_env.manager.get_current_version().await;
+    let old_cg = new_version
+        .levels
+        .get(&(StaticCompactionGroupId::StateDefault as _))
+        .unwrap();
+    let sub_levels = &old_cg.l0.sub_levels;
+    assert_eq!(sub_levels.len(), 2);
+    let sub_level1 = &sub_levels[0];
+    assert_eq!(sub_level1.sub_level_id, epoch1);
+    assert_eq!(sub_level1.table_infos.len(), 1);
+    assert_eq!(sub_level1.table_infos[0].object_id, sst1_epoch1.object_id);
+    let sub_level2 = &sub_levels[1];
+    assert_eq!(sub_level2.sub_level_id, epoch2);
+    assert_eq!(sub_level2.table_infos.len(), 1);
+    assert_eq!(sub_level2.table_infos[0].object_id, sst_epoch2.object_id);
+
+    let new_cg_id_set: HashSet<_> = new_version.levels.keys().cloned().collect();
+    let added_cg_id_set = &new_cg_id_set - &old_cg_id_set;
+    assert_eq!(added_cg_id_set.len(), 1);
+    let new_cg_id = added_cg_id_set.into_iter().next().unwrap();
+
+    let new_cg = new_version.levels.get(&new_cg_id).unwrap();
+    let sub_levels = &new_cg.l0.sub_levels;
+    assert_eq!(sub_levels.len(), 2);
+    let sub_level1 = &sub_levels[0];
+    assert_eq!(sub_level1.sub_level_id, epoch1);
+    assert_eq!(sub_level1.table_infos.len(), 1);
+    assert_eq!(sub_level1.table_infos[0].object_id, sst2_epoch1.object_id);
+    let sub_level2 = &sub_levels[1];
+    assert_eq!(sub_level2.sub_level_id, epoch2);
+    assert_eq!(sub_level2.table_infos.len(), 1);
+    assert_eq!(sub_level2.table_infos[0].object_id, sst_epoch2.object_id);
+
+    let info = new_version
+        .state_table_info
+        .info()
+        .get(&batch_commit_table_id)
+        .unwrap();
+    assert_eq!(info.committed_epoch, epoch2);
+    assert_eq!(info.compaction_group_id, new_cg_id);
+}
diff --git a/src/storage/hummock_test/src/mock_notification_client.rs b/src/storage/hummock_test/src/mock_notification_client.rs
index 68c75d57937a2..0de4805bdef75 100644
--- a/src/storage/hummock_test/src/mock_notification_client.rs
+++ b/src/storage/hummock_test/src/mock_notification_client.rs
@@ -16,7 +16,7 @@ use std::collections::HashMap;
 use std::sync::Arc;
 
 use risingwave_common::util::addr::HostAddr;
-use risingwave_common_service::observer_manager::{Channel, NotificationClient, ObserverError};
+use risingwave_common_service::{Channel, NotificationClient, ObserverError};
 use risingwave_meta::hummock::{HummockManager, HummockManagerRef};
 use risingwave_meta::manager::{MessageStatus, MetaSrvEnv, NotificationManagerRef, WorkerKey};
 use risingwave_pb::backup_service::MetaBackupManifestId;
@@ -62,7 +62,7 @@ impl NotificationClient for MockNotificationClient {
 
         let hummock_version = self.hummock_manager.get_current_version().await;
         let meta_snapshot = MetaSnapshot {
-            hummock_version: Some(hummock_version.to_protobuf()),
+            hummock_version: Some(hummock_version.into()),
             version: Some(Default::default()),
             meta_backup_manifest_id: Some(MetaBackupManifestId { id: 0 }),
             hummock_write_limits: Some(WriteLimits {
diff --git a/src/storage/hummock_test/src/snapshot_tests.rs b/src/storage/hummock_test/src/snapshot_tests.rs
index 402952dd09680..bde3c046ed6ca 100644
--- a/src/storage/hummock_test/src/snapshot_tests.rs
+++ b/src/storage/hummock_test/src/snapshot_tests.rs
@@ -1,3 +1,4 @@
+use std::collections::HashSet;
 // Copyright 2024 RisingWave Labs
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
@@ -113,6 +114,7 @@ async fn test_snapshot_inner(
         .await;
 
     let epoch1 = test_epoch(1);
+    hummock_storage.start_epoch(epoch1, HashSet::from_iter([Default::default()]));
     local.init_for_test(epoch1).await.unwrap();
     local
         .ingest_batch(
@@ -134,6 +136,7 @@ async fn test_snapshot_inner(
         .await
         .unwrap();
     let epoch2 = epoch1.next_epoch();
+    hummock_storage.start_epoch(epoch2, HashSet::from_iter([Default::default()]));
     local.seal_current_epoch(epoch2, SealCurrentEpochOptions::for_test());
     if enable_sync {
         let res = hummock_storage.seal_and_sync_epoch(epoch1).await.unwrap();
@@ -174,6 +177,7 @@ async fn test_snapshot_inner(
         .await
         .unwrap();
     let epoch3 = epoch2.next_epoch();
+    hummock_storage.start_epoch(epoch3, HashSet::from_iter([Default::default()]));
     local.seal_current_epoch(epoch3, SealCurrentEpochOptions::for_test());
     if enable_sync {
         let res = hummock_storage.seal_and_sync_epoch(epoch2).await.unwrap();
@@ -243,6 +247,7 @@ async fn test_snapshot_range_scan_inner(
     let mut local = hummock_storage
         .new_local(NewLocalOptions::for_test(Default::default()))
         .await;
+    hummock_storage.start_epoch(epoch, HashSet::from_iter([Default::default()]));
     local.init_for_test(epoch).await.unwrap();
 
     local
diff --git a/src/storage/hummock_test/src/state_store_tests.rs b/src/storage/hummock_test/src/state_store_tests.rs
index fe129deb52245..8df4170722072 100644
--- a/src/storage/hummock_test/src/state_store_tests.rs
+++ b/src/storage/hummock_test/src/state_store_tests.rs
@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::collections::HashSet;
 use std::ops::Bound;
 use std::ops::Bound::{Excluded, Included, Unbounded};
 use std::sync::Arc;
@@ -21,11 +22,11 @@ use expect_test::expect;
 use foyer::CacheContext;
 use futures::{pin_mut, StreamExt};
 use itertools::Itertools;
-use risingwave_common::buffer::Bitmap;
+use risingwave_common::bitmap::Bitmap;
 use risingwave_common::catalog::{TableId, TableOption};
 use risingwave_common::hash::table_distribution::TableDistribution;
 use risingwave_common::hash::VirtualNode;
-use risingwave_common::util::epoch::{test_epoch, EpochExt};
+use risingwave_common::util::epoch::{test_epoch, EpochExt, MAX_EPOCH};
 use risingwave_hummock_sdk::key::{prefixed_range_with_vnode, TableKeyRange};
 use risingwave_hummock_sdk::{
     HummockReadEpoch, HummockSstableObjectId, LocalSstableInfo, SyncResult,
@@ -133,6 +134,7 @@ async fn test_basic_v2() {
 
     // epoch 0 is reserved by storage service
     let epoch1 = test_epoch(1);
+    hummock_storage.start_epoch(epoch1, HashSet::from_iter([Default::default()]));
     local.init_for_test(epoch1).await.unwrap();
 
     // try to write an empty batch, and hummock should write nothing
@@ -162,6 +164,7 @@ async fn test_basic_v2() {
         .unwrap();
 
     let epoch2 = epoch1.next_epoch();
+    hummock_storage.start_epoch(epoch2, HashSet::from_iter([Default::default()]));
     local.seal_current_epoch(epoch2, SealCurrentEpochOptions::for_test());
 
     // Get the value after flushing to remote.
@@ -219,6 +222,7 @@ async fn test_basic_v2() {
         .unwrap();
 
     let epoch3 = epoch2.next_epoch();
+    hummock_storage.start_epoch(epoch3, HashSet::from_iter([Default::default()]));
     local.seal_current_epoch(epoch3, SealCurrentEpochOptions::for_test());
 
     // Get the value after flushing to remote.
@@ -432,6 +436,7 @@ async fn test_state_store_sync_v2() {
     let mut local = hummock_storage
         .new_local(NewLocalOptions::for_test(Default::default()))
         .await;
+    hummock_storage.start_epoch(epoch, HashSet::from_iter([Default::default()]));
     local.init_for_test(epoch).await.unwrap();
     local
         .ingest_batch(
@@ -481,6 +486,7 @@ async fn test_state_store_sync_v2() {
     // );
 
     epoch.inc_epoch();
+    hummock_storage.start_epoch(epoch, HashSet::from_iter([Default::default()]));
     local.seal_current_epoch(epoch, SealCurrentEpochOptions::for_test());
 
     // ingest more 8B then will trigger a sync behind the scene
@@ -1022,6 +1028,7 @@ async fn test_delete_get_v2() {
 
     let initial_epoch = hummock_storage.get_pinned_version().max_committed_epoch();
     let epoch1 = initial_epoch.next_epoch();
+    hummock_storage.start_epoch(epoch1, HashSet::from_iter([Default::default()]));
     let batch1 = vec![
         (
             gen_key_from_str(VirtualNode::ZERO, "aa"),
@@ -1047,6 +1054,7 @@ async fn test_delete_get_v2() {
         .await
         .unwrap();
     let epoch2 = epoch1.next_epoch();
+    hummock_storage.start_epoch(epoch2, HashSet::from_iter([Default::default()]));
 
     local.seal_current_epoch(epoch2, SealCurrentEpochOptions::for_test());
     let res = hummock_storage.seal_and_sync_epoch(epoch1).await.unwrap();
@@ -1107,6 +1115,7 @@ async fn test_multiple_epoch_sync_v2() {
     let mut local = hummock_storage
         .new_local(NewLocalOptions::for_test(TableId::default()))
         .await;
+    hummock_storage.start_epoch(epoch1, HashSet::from_iter([Default::default()]));
     local.init_for_test(epoch1).await.unwrap();
     local
         .ingest_batch(
@@ -1120,6 +1129,7 @@ async fn test_multiple_epoch_sync_v2() {
         .unwrap();
 
     let epoch2 = epoch1.next_epoch();
+    hummock_storage.start_epoch(epoch2, HashSet::from_iter([Default::default()]));
     local.seal_current_epoch(epoch2, SealCurrentEpochOptions::for_test());
     let batch2 = vec![(
         gen_key_from_str(VirtualNode::ZERO, "bb"),
@@ -1137,6 +1147,7 @@ async fn test_multiple_epoch_sync_v2() {
         .unwrap();
 
     let epoch3 = epoch2.next_epoch();
+    hummock_storage.start_epoch(epoch3, HashSet::from_iter([Default::default()]));
     let batch3 = vec![
         (
             gen_key_from_str(VirtualNode::ZERO, "aa"),
@@ -1245,6 +1256,7 @@ async fn test_gc_watermark_and_clear_shared_buffer() {
 
     let initial_epoch = hummock_storage.get_pinned_version().max_committed_epoch();
     let epoch1 = initial_epoch.next_epoch();
+    hummock_storage.start_epoch(epoch1, HashSet::from_iter([Default::default()]));
     local_hummock_storage.init_for_test(epoch1).await.unwrap();
     local_hummock_storage
         .insert(
@@ -1270,6 +1282,7 @@ async fn test_gc_watermark_and_clear_shared_buffer() {
     );
 
     let epoch2 = epoch1.next_epoch();
+    hummock_storage.start_epoch(epoch2, HashSet::from_iter([Default::default()]));
     local_hummock_storage.seal_current_epoch(epoch2, SealCurrentEpochOptions::for_test());
     local_hummock_storage
         .delete(
@@ -1289,7 +1302,7 @@ async fn test_gc_watermark_and_clear_shared_buffer() {
         sync_result
             .uncommitted_ssts
             .iter()
-            .map(|LocalSstableInfo { sst_info, .. }| sst_info.get_object_id())
+            .map(|LocalSstableInfo { sst_info, .. }| sst_info.object_id)
             .min()
             .unwrap()
     };
@@ -1536,6 +1549,10 @@ async fn test_iter_log() {
     let key_count = 10000;
 
     let test_log_data = gen_test_data(epoch_count, key_count, 0.05, 0.2);
+    for (epoch, _) in &test_log_data {
+        hummock_storage.start_epoch(*epoch, HashSet::from_iter([table_id]));
+    }
+    hummock_storage.start_epoch(MAX_EPOCH, HashSet::from_iter([table_id]));
     let in_memory_state_store = MemoryStateStore::new();
 
     let mut in_memory_local = in_memory_state_store
diff --git a/src/storage/hummock_test/src/sync_point_tests.rs b/src/storage/hummock_test/src/sync_point_tests.rs
index 3074bba6dd75f..f5ee41783813d 100644
--- a/src/storage/hummock_test/src/sync_point_tests.rs
+++ b/src/storage/hummock_test/src/sync_point_tests.rs
@@ -213,7 +213,7 @@ pub async fn compact_once(
     hummock_manager_ref
         .report_compact_task(
             result_task.task_id,
-            result_task.task_status(),
+            result_task.task_status,
             result_task.sorted_output_ssts,
             Some(to_prost_table_stats_map(task_stats)),
         )
@@ -417,16 +417,10 @@ async fn test_syncpoints_get_in_delete_range_boundary() {
         .get_compaction_group_levels(StaticCompactionGroupId::StateDefault.into())
         .levels[4];
     assert_eq!(base_level.table_infos.len(), 3);
-    assert!(
-        base_level.table_infos[0]
-            .key_range
-            .as_ref()
-            .unwrap()
-            .right_exclusive
-    );
+    assert!(base_level.table_infos[0].key_range.right_exclusive);
     assert_eq!(
-        user_key(&base_level.table_infos[0].key_range.as_ref().unwrap().right),
-        user_key(&base_level.table_infos[1].key_range.as_ref().unwrap().left),
+        user_key(&base_level.table_infos[0].key_range.right),
+        user_key(&base_level.table_infos[1].key_range.left),
     );
     storage.wait_version(version).await;
     let read_options = ReadOptions {
diff --git a/src/storage/hummock_test/src/test_utils.rs b/src/storage/hummock_test/src/test_utils.rs
index 5a2034facdbfa..bf5c4a8dd8d8c 100644
--- a/src/storage/hummock_test/src/test_utils.rs
+++ b/src/storage/hummock_test/src/test_utils.rs
@@ -17,7 +17,7 @@ use std::sync::Arc;
 use bytes::Bytes;
 use itertools::Itertools;
 use risingwave_common::catalog::TableId;
-use risingwave_common_service::observer_manager::ObserverManager;
+use risingwave_common_service::ObserverManager;
 use risingwave_hummock_sdk::compaction_group::StaticCompactionGroupId;
 use risingwave_hummock_sdk::key::TableKey;
 pub use risingwave_hummock_sdk::key::{gen_key_from_bytes, gen_key_from_str};
diff --git a/src/storage/hummock_test/src/vacuum_tests.rs b/src/storage/hummock_test/src/vacuum_tests.rs
index 5c6dbdc556d5a..5c6dbd20587cc 100644
--- a/src/storage/hummock_test/src/vacuum_tests.rs
+++ b/src/storage/hummock_test/src/vacuum_tests.rs
@@ -92,6 +92,7 @@ async fn test_full_scan() {
 
     let task = FullScanTask {
         sst_retention_time_sec: 10000,
+        prefix: None,
     };
     let (scan_result, _, _) = Vacuum::full_scan_inner(task, object_metadata_iter.clone())
         .await
@@ -100,6 +101,7 @@ async fn test_full_scan() {
 
     let task = FullScanTask {
         sst_retention_time_sec: 6000,
+        prefix: None,
     };
     let (scan_result, _, _) = Vacuum::full_scan_inner(task, object_metadata_iter.clone())
         .await
@@ -108,6 +110,7 @@ async fn test_full_scan() {
 
     let task = FullScanTask {
         sst_retention_time_sec: 2000,
+        prefix: None,
     };
     let (scan_result, _, _) = Vacuum::full_scan_inner(task, object_metadata_iter)
         .await
diff --git a/src/storage/hummock_trace/src/lib.rs b/src/storage/hummock_trace/src/lib.rs
index 64417832206e0..48b0a71010a74 100644
--- a/src/storage/hummock_trace/src/lib.rs
+++ b/src/storage/hummock_trace/src/lib.rs
@@ -12,7 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#![feature(lazy_cell)]
 #![feature(cursor_remaining)]
 #![feature(trait_alias)]
 #![feature(coroutines)]
diff --git a/src/storage/hummock_trace/src/opts.rs b/src/storage/hummock_trace/src/opts.rs
index 07c87f4688042..2a5b62f046d32 100644
--- a/src/storage/hummock_trace/src/opts.rs
+++ b/src/storage/hummock_trace/src/opts.rs
@@ -14,7 +14,7 @@
 
 use bincode::{Decode, Encode};
 use foyer::CacheContext;
-use risingwave_common::buffer::Bitmap;
+use risingwave_common::bitmap::Bitmap;
 use risingwave_common::cache::CachePriority;
 use risingwave_common::catalog::{TableId, TableOption};
 use risingwave_common::util::epoch::EpochPair;
@@ -109,6 +109,7 @@ pub struct TracedReadOptions {
     pub retention_seconds: Option<u32>,
     pub table_id: TracedTableId,
     pub read_version_from_backup: bool,
+    pub read_version_from_time_travel: bool,
 }
 
 impl TracedReadOptions {
@@ -123,7 +124,8 @@ impl TracedReadOptions {
             cache_policy: TracedCachePolicy::Disable,
             retention_seconds: None,
             table_id: TracedTableId { table_id },
-            read_version_from_backup: true,
+            read_version_from_backup: false,
+            read_version_from_time_travel: false,
         }
     }
 }
@@ -195,6 +197,7 @@ pub enum TracedHummockReadEpoch {
     Current(TracedHummockEpoch),
     NoWait(TracedHummockEpoch),
     Backup(TracedHummockEpoch),
+    TimeTravel(TracedHummockEpoch),
 }
 
 impl From<HummockReadEpoch> for TracedHummockReadEpoch {
@@ -204,6 +207,7 @@ impl From<HummockReadEpoch> for TracedHummockReadEpoch {
             HummockReadEpoch::Current(epoch) => Self::Current(epoch),
             HummockReadEpoch::NoWait(epoch) => Self::NoWait(epoch),
             HummockReadEpoch::Backup(epoch) => Self::Backup(epoch),
+            HummockReadEpoch::TimeTravel(epoch) => Self::TimeTravel(epoch),
         }
     }
 }
@@ -215,6 +219,7 @@ impl From<TracedHummockReadEpoch> for HummockReadEpoch {
             TracedHummockReadEpoch::Current(epoch) => Self::Current(epoch),
             TracedHummockReadEpoch::NoWait(epoch) => Self::NoWait(epoch),
             TracedHummockReadEpoch::Backup(epoch) => Self::Backup(epoch),
+            TracedHummockReadEpoch::TimeTravel(epoch) => Self::TimeTravel(epoch),
         }
     }
 }
diff --git a/src/storage/src/hummock/compactor/compaction_utils.rs b/src/storage/src/hummock/compactor/compaction_utils.rs
index 63b59366195f0..6eb49d627b340 100644
--- a/src/storage/src/hummock/compactor/compaction_utils.rs
+++ b/src/storage/src/hummock/compactor/compaction_utils.rs
@@ -18,18 +18,18 @@ use std::ops::Bound;
 use std::sync::atomic::{AtomicU64, Ordering};
 use std::sync::Arc;
 
+use bytes::Bytes;
 use itertools::Itertools;
 use risingwave_common::constants::hummock::CompactionFilterFlag;
+use risingwave_hummock_sdk::compact_task::CompactTask;
 use risingwave_hummock_sdk::compaction_group::StateTableId;
 use risingwave_hummock_sdk::key::FullKey;
 use risingwave_hummock_sdk::key_range::KeyRange;
-use risingwave_hummock_sdk::prost_key_range::KeyRangeExt;
+use risingwave_hummock_sdk::sstable_info::SstableInfo;
 use risingwave_hummock_sdk::table_stats::TableStatsMap;
 use risingwave_hummock_sdk::{can_concat, EpochWithGap, KeyComparator};
-use risingwave_pb::hummock::compact_task::TaskType;
-use risingwave_pb::hummock::{
-    compact_task, BloomFilterType, CompactTask, LevelType, PbKeyRange, SstableInfo, TableSchema,
-};
+use risingwave_pb::hummock::compact_task::PbTaskType;
+use risingwave_pb::hummock::{BloomFilterType, PbLevelType, PbTableSchema};
 use tokio::time::Instant;
 
 pub use super::context::CompactorContext;
@@ -127,7 +127,7 @@ pub struct TaskConfig {
     /// change. For an divided SST as input, a dropped key shouldn't be counted if its table id
     /// doesn't belong to this divided SST. See `Compactor::compact_and_build_sst`.
     pub stats_target_table_ids: Option<HashSet<u32>>,
-    pub task_type: compact_task::TaskType,
+    pub task_type: PbTaskType,
     pub is_target_l0_or_lbase: bool,
     pub use_block_based_filter: bool,
 
@@ -135,7 +135,7 @@ pub struct TaskConfig {
     /// `TableId` -> `TableSchema`
     /// Schemas in `table_schemas` are at least as new as the one used to create `input_ssts`.
     /// For a table with schema existing in `table_schemas`, its columns not in `table_schemas` but in `input_ssts` can be safely dropped.
-    pub table_schemas: HashMap<u32, TableSchema>,
+    pub table_schemas: HashMap<u32, PbTableSchema>,
     /// `disable_drop_column_optimization` should only be set in benchmark.
     pub disable_drop_column_optimization: bool,
 }
@@ -178,7 +178,7 @@ fn generate_splits_fast(
     compaction_size: u64,
     context: &CompactorContext,
     max_sub_compaction: u32,
-) -> Vec<PbKeyRange> {
+) -> Vec<KeyRange> {
     let worker_num = context.compaction_executor.worker_num();
     let parallel_compact_size = (context.storage_opts.parallel_compact_size_mb as u64) << 20;
 
@@ -190,7 +190,7 @@ fn generate_splits_fast(
     );
     let mut indexes = vec![];
     for sst in sstable_infos {
-        let key_range = sst.key_range.as_ref().unwrap();
+        let key_range = &sst.key_range;
         indexes.push(
             FullKey {
                 user_key: FullKey::decode(&key_range.left).user_key,
@@ -213,13 +213,13 @@ fn generate_splits_fast(
     }
 
     let mut splits = vec![];
-    splits.push(PbKeyRange::new(vec![], vec![]));
+    splits.push(KeyRange::default());
     let parallel_key_count = indexes.len() / parallelism;
     let mut last_split_key_count = 0;
     for key in indexes {
         if last_split_key_count >= parallel_key_count {
-            splits.last_mut().unwrap().right.clone_from(&key);
-            splits.push(PbKeyRange::new(key.clone(), vec![]));
+            splits.last_mut().unwrap().right = Bytes::from(key.clone());
+            splits.push(KeyRange::new(Bytes::from(key.clone()), Bytes::default()));
             last_split_key_count = 0;
         }
         last_split_key_count += 1;
@@ -233,7 +233,7 @@ pub async fn generate_splits(
     compaction_size: u64,
     context: &CompactorContext,
     max_sub_compaction: u32,
-) -> HummockResult<Vec<PbKeyRange>> {
+) -> HummockResult<Vec<KeyRange>> {
     const MAX_FILE_COUNT: usize = 32;
     let parallel_compact_size = (context.storage_opts.parallel_compact_size_mb as u64) << 20;
     if compaction_size > parallel_compact_size {
@@ -271,7 +271,7 @@ pub async fn generate_splits(
         // sort by key, as for every data block has the same size;
         indexes.sort_by(|a, b| KeyComparator::compare_encoded_full_key(a.1.as_ref(), b.1.as_ref()));
         let mut splits = vec![];
-        splits.push(PbKeyRange::new(vec![], vec![]));
+        splits.push(KeyRange::default());
 
         let parallelism = calculate_task_parallelism_impl(
             context.compaction_executor.worker_num(),
@@ -292,8 +292,8 @@ pub async fn generate_splits(
                     && !last_key.eq(&key)
                     && remaining_size > parallel_compact_size
                 {
-                    splits.last_mut().unwrap().right.clone_from(&key);
-                    splits.push(PbKeyRange::new(key.clone(), vec![]));
+                    splits.last_mut().unwrap().right = Bytes::from(key.clone());
+                    splits.push(KeyRange::new(Bytes::from(key.clone()), Bytes::default()));
                     last_buffer_size = data_size;
                 } else {
                     last_buffer_size += data_size;
@@ -357,7 +357,7 @@ pub async fn check_compaction_result(
         }
 
         // Do not need to filter the table because manager has done it.
-        if level.level_type == LevelType::Nonoverlapping as i32 {
+        if level.level_type == PbLevelType::Nonoverlapping {
             debug_assert!(can_concat(&level.table_infos));
             del_iter.add_concat_iter(level.table_infos.clone(), context.sstable_store.clone());
 
@@ -513,7 +513,7 @@ pub fn optimize_by_copy_block(compact_task: &CompactTask, context: &CompactorCon
 
     let all_ssts_are_blocked_filter = sstable_infos
         .iter()
-        .all(|table_info| table_info.bloom_filter_kind() == BloomFilterType::Blocked);
+        .all(|table_info| table_info.bloom_filter_kind == BloomFilterType::Blocked);
 
     let delete_key_count = sstable_infos
         .iter()
@@ -553,7 +553,7 @@ pub fn optimize_by_copy_block(compact_task: &CompactTask, context: &CompactorCon
         && compaction_size < context.storage_opts.compactor_fast_max_compact_task_size
         && delete_key_count * 100
             < context.storage_opts.compactor_fast_max_compact_delete_ratio as u64 * total_key_count
-        && compact_task.task_type() == TaskType::Dynamic
+        && compact_task.task_type == PbTaskType::Dynamic
 }
 
 pub async fn generate_splits_for_task(
@@ -583,7 +583,7 @@ pub async fn generate_splits_for_task(
             &sstable_infos,
             compaction_size,
             context,
-            compact_task.get_max_sub_compaction(),
+            compact_task.max_sub_compaction,
         )
         .await?;
         if !splits.is_empty() {
@@ -667,7 +667,7 @@ pub fn calculate_task_parallelism(compact_task: &CompactTask, context: &Compacto
         context.compaction_executor.worker_num(),
         parallel_compact_size,
         compaction_size,
-        compact_task.get_max_sub_compaction(),
+        compact_task.max_sub_compaction,
     )
 }
 
diff --git a/src/storage/src/hummock/compactor/compactor_runner.rs b/src/storage/src/hummock/compactor/compactor_runner.rs
index 0336f6f542fa4..773b2d565550e 100644
--- a/src/storage/src/hummock/compactor/compactor_runner.rs
+++ b/src/storage/src/hummock/compactor/compactor_runner.rs
@@ -23,14 +23,16 @@ use risingwave_common::util::value_encoding::column_aware_row_encoding::try_drop
 use risingwave_hummock_sdk::compact::{
     compact_task_to_string, estimate_memory_for_compact_task, statistics_compact_task,
 };
+use risingwave_hummock_sdk::compact_task::CompactTask;
 use risingwave_hummock_sdk::key::{FullKey, FullKeyTracker};
 use risingwave_hummock_sdk::key_range::{KeyRange, KeyRangeCommon};
+use risingwave_hummock_sdk::sstable_info::SstableInfo;
 use risingwave_hummock_sdk::table_stats::{add_table_stats_map, TableStats, TableStatsMap};
 use risingwave_hummock_sdk::{
     can_concat, compact_task_output_to_string, HummockSstableObjectId, KeyComparator,
 };
 use risingwave_pb::hummock::compact_task::TaskStatus;
-use risingwave_pb::hummock::{CompactTask, LevelType, SstableInfo};
+use risingwave_pb::hummock::LevelType;
 use thiserror_ext::AsReport;
 use tokio::sync::oneshot::Receiver;
 
@@ -94,8 +96,8 @@ impl CompactorRunner {
             BlockedXor16FilterBuilder::is_kv_count_too_large(kv_count) || task.target_level > 0;
 
         let key_range = KeyRange {
-            left: Bytes::copy_from_slice(task.splits[split_index].get_left()),
-            right: Bytes::copy_from_slice(task.splits[split_index].get_right()),
+            left: task.splits[split_index].left.clone(),
+            right: task.splits[split_index].right.clone(),
             right_exclusive: true,
         };
 
@@ -108,7 +110,7 @@ impl CompactorRunner {
                 gc_delete_keys: task.gc_delete_keys,
                 watermark: task.watermark,
                 stats_target_table_ids: Some(HashSet::from_iter(task.existing_table_ids.clone())),
-                task_type: task.task_type(),
+                task_type: task.task_type,
                 is_target_l0_or_lbase: task.target_level == 0
                     || task.target_level == task.base_level,
                 use_block_based_filter,
@@ -173,18 +175,17 @@ impl CompactorRunner {
                 .table_infos
                 .iter()
                 .filter(|table_info| {
-                    let key_range = KeyRange::from(table_info.key_range.as_ref().unwrap());
                     let table_ids = &table_info.table_ids;
                     let exist_table = table_ids
                         .iter()
                         .any(|table_id| self.compact_task.existing_table_ids.contains(table_id));
 
-                    self.key_range.full_key_overlap(&key_range) && exist_table
+                    self.key_range.full_key_overlap(&table_info.key_range) && exist_table
                 })
                 .cloned()
                 .collect_vec();
             // Do not need to filter the table because manager has done it.
-            if level.level_type == LevelType::Nonoverlapping as i32 {
+            if level.level_type == LevelType::Nonoverlapping {
                 debug_assert!(can_concat(&level.table_infos));
                 table_iters.push(ConcatSstableIterator::new(
                     self.compact_task.existing_table_ids.clone(),
@@ -202,7 +203,7 @@ impl CompactorRunner {
                     sst_groups.len()
                 );
                 for table_infos in sst_groups {
-                    assert!(can_concat(&table_infos));
+                    assert!(can_concat(table_infos.as_slice()));
                     table_iters.push(ConcatSstableIterator::new(
                         self.compact_task.existing_table_ids.clone(),
                         table_infos,
@@ -243,7 +244,7 @@ pub fn partition_overlapping_sstable_infos(
 ) -> Vec<Vec<SstableInfo>> {
     pub struct SstableGroup {
         ssts: Vec<SstableInfo>,
-        max_right_bound: Vec<u8>,
+        max_right_bound: Bytes,
     }
 
     impl PartialEq for SstableGroup {
@@ -265,8 +266,8 @@ pub fn partition_overlapping_sstable_infos(
     }
     let mut groups: BinaryHeap<SstableGroup> = BinaryHeap::default();
     origin_infos.sort_by(|a, b| {
-        let x = a.key_range.as_ref().unwrap();
-        let y = b.key_range.as_ref().unwrap();
+        let x = &a.key_range;
+        let y = &b.key_range;
         KeyComparator::compare_encoded_full_key(&x.left, &y.left)
     });
     for sst in origin_infos {
@@ -274,17 +275,15 @@ pub fn partition_overlapping_sstable_infos(
         if let Some(mut prev_group) = groups.peek_mut() {
             if KeyComparator::encoded_full_key_less_than(
                 &prev_group.max_right_bound,
-                &sst.key_range.as_ref().unwrap().left,
+                &sst.key_range.left,
             ) {
-                prev_group
-                    .max_right_bound
-                    .clone_from(&sst.key_range.as_ref().unwrap().right);
+                prev_group.max_right_bound.clone_from(&sst.key_range.right);
                 prev_group.ssts.push(sst);
                 continue;
             }
         }
         groups.push(SstableGroup {
-            max_right_bound: sst.key_range.as_ref().unwrap().right.clone(),
+            max_right_bound: sst.key_range.right.clone(),
             ssts: vec![sst],
         });
     }
@@ -577,7 +576,7 @@ pub(crate) fn compact_done(
     task_status: TaskStatus,
 ) -> (CompactTask, HashMap<u32, TableStats>) {
     let mut table_stats_map = TableStatsMap::default();
-    compact_task.set_task_status(task_status);
+    compact_task.task_status = task_status;
     compact_task
         .sorted_output_ssts
         .reserve(compact_task.splits.len());
diff --git a/src/storage/src/hummock/compactor/fast_compactor_runner.rs b/src/storage/src/hummock/compactor/fast_compactor_runner.rs
index 81f0210d945c1..227a50cb5813d 100644
--- a/src/storage/src/hummock/compactor/fast_compactor_runner.rs
+++ b/src/storage/src/hummock/compactor/fast_compactor_runner.rs
@@ -22,11 +22,12 @@ use std::time::Instant;
 use await_tree::InstrumentAwait;
 use bytes::Bytes;
 use itertools::Itertools;
+use risingwave_hummock_sdk::compact_task::CompactTask;
 use risingwave_hummock_sdk::key::FullKey;
 use risingwave_hummock_sdk::key_range::KeyRange;
+use risingwave_hummock_sdk::sstable_info::SstableInfo;
 use risingwave_hummock_sdk::table_stats::TableStats;
 use risingwave_hummock_sdk::{can_concat, compact_task_to_string, EpochWithGap, LocalSstableInfo};
-use risingwave_pb::hummock::{CompactTask, SstableInfo};
 
 use crate::filter_key_extractor::FilterKeyExtractorImpl;
 use crate::hummock::block_stream::BlockDataStream;
@@ -292,7 +293,7 @@ impl CompactorRunner {
             gc_delete_keys: task.gc_delete_keys,
             watermark: task.watermark,
             stats_target_table_ids: Some(HashSet::from_iter(task.existing_table_ids.clone())),
-            task_type: task.task_type(),
+            task_type: task.task_type,
             is_target_l0_or_lbase: task.target_level == 0 || task.target_level == task.base_level,
             table_vnode_partition: task.table_vnode_partition.clone(),
             use_block_based_filter: true,
diff --git a/src/storage/src/hummock/compactor/iterator.rs b/src/storage/src/hummock/compactor/iterator.rs
index 5431c51270efc..84fa03c7b71d6 100644
--- a/src/storage/src/hummock/compactor/iterator.rs
+++ b/src/storage/src/hummock/compactor/iterator.rs
@@ -23,8 +23,8 @@ use fail::fail_point;
 use risingwave_hummock_sdk::compaction_group::StateTableId;
 use risingwave_hummock_sdk::key::FullKey;
 use risingwave_hummock_sdk::key_range::KeyRange;
+use risingwave_hummock_sdk::sstable_info::SstableInfo;
 use risingwave_hummock_sdk::KeyComparator;
-use risingwave_pb::hummock::SstableInfo;
 
 use crate::hummock::block_stream::BlockDataStream;
 use crate::hummock::compactor::task_progress::TaskProgress;
@@ -37,8 +37,11 @@ use crate::monitor::StoreLocalStatistic;
 const PROGRESS_KEY_INTERVAL: usize = 100;
 
 /// Iterates over the KV-pairs of an SST while downloading it.
+/// `SstableStreamIterator` encapsulates operations on `sstables`, constructing block streams and accessing the corresponding data via `block_metas`.
+///  Note that a `block_meta` does not necessarily correspond to the entire sstable, but rather to a subset, which is documented via the `block_idx`.
 pub struct SstableStreamIterator {
     sstable_store: SstableStoreRef,
+    /// The block metas subset of the SST.
     block_metas: Vec<BlockMeta>,
     /// The downloading stream.
     block_stream: Option<BlockDataStream>,
@@ -46,7 +49,8 @@ pub struct SstableStreamIterator {
     /// Iterates over the KV-pairs of the current block.
     block_iter: Option<BlockIterator>,
 
-    seek_block_idx: usize,
+    /// Index of the current block.
+    block_idx: usize,
 
     /// Counts the time used for IO.
     stats_ptr: Arc<AtomicU64>,
@@ -78,7 +82,6 @@ impl SstableStreamIterator {
         block_metas: Vec<BlockMeta>,
         sstable_info: SstableInfo,
         existing_table_ids: HashSet<StateTableId>,
-        start_block_idx: usize,
         stats: &StoreLocalStatistic,
         task_progress: Arc<TaskProgress>,
         sstable_store: SstableStoreRef,
@@ -88,7 +91,7 @@ impl SstableStreamIterator {
             block_stream: None,
             block_iter: None,
             block_metas,
-            seek_block_idx: start_block_idx,
+            block_idx: 0,
             stats_ptr: stats.remote_io_time.clone(),
             existing_table_ids,
             sstable_info,
@@ -104,7 +107,7 @@ impl SstableStreamIterator {
             .sstable_store
             .get_stream_for_blocks(
                 self.sstable_info.object_id,
-                &self.block_metas[self.seek_block_idx..],
+                &self.block_metas[self.block_idx..],
             )
             .verbose_instrument_await("stream_iter_get_stream")
             .await?;
@@ -156,7 +159,7 @@ impl SstableStreamIterator {
     /// `self.block_iter` to `None`.
     async fn next_block(&mut self) -> HummockResult<()> {
         // Check if we want and if we can load the next block.
-        if self.seek_block_idx < self.block_metas.len() {
+        if self.block_idx < self.block_metas.len() {
             loop {
                 let now = Instant::now();
                 let ret = match &mut self.block_stream {
@@ -172,7 +175,7 @@ impl SstableStreamIterator {
                         let mut block_iter =
                             BlockIterator::new(BlockHolder::from_owned_block(block));
                         block_iter.seek_to_first();
-                        self.seek_block_idx += 1;
+                        self.block_idx += 1;
                         self.block_iter = Some(block_iter);
                         return Ok(());
                     }
@@ -190,7 +193,7 @@ impl SstableStreamIterator {
                     .fetch_add(add as u64, atomic::Ordering::Relaxed);
             }
         }
-        self.seek_block_idx = self.block_metas.len();
+        self.block_idx = self.block_metas.len();
         self.block_iter = None;
 
         Ok(())
@@ -242,8 +245,8 @@ impl SstableStreamIterator {
     fn sst_debug_info(&self) -> String {
         format!(
             "object_id={}, sst_id={}, meta_offset={}, table_ids={:?}",
-            self.sstable_info.get_object_id(),
-            self.sstable_info.get_sst_id(),
+            self.sstable_info.object_id,
+            self.sstable_info.sst_id,
             self.sstable_info.meta_offset,
             self.sstable_info.table_ids
         )
@@ -344,6 +347,7 @@ impl ConcatSstableIterator {
             (None, true) => None,
             (None, false) => Some(FullKey::decode(&self.key_range.left)),
         };
+
         self.cur_idx = idx;
         while self.cur_idx < self.sstables.len() {
             let table_info = &self.sstables[self.cur_idx];
@@ -361,49 +365,28 @@ impl ConcatSstableIterator {
                 .sstable(table_info, &mut self.stats)
                 .verbose_instrument_await("stream_iter_sstable")
                 .await?;
-            let block_metas = &sstable.meta.block_metas;
-            let mut start_index = match seek_key {
-                None => 0,
+
+            let filter_key_range = match seek_key {
                 Some(seek_key) => {
-                    // start_index points to the greatest block whose smallest_key <= seek_key.
-                    block_metas
-                        .partition_point(|block| {
-                            seek_key.cmp(&FullKey::decode(&block.smallest_key)) != Ordering::Less
-                        })
-                        .saturating_sub(1)
+                    KeyRange::new(seek_key.encode().into(), self.key_range.right.clone())
                 }
+                None => self.key_range.clone(),
             };
-            let end_index = if self.key_range.right.is_empty() {
-                block_metas.len()
-            } else {
-                block_metas.partition_point(|block| {
-                    KeyComparator::compare_encoded_full_key(
-                        &block.smallest_key,
-                        &self.key_range.right,
-                    ) != Ordering::Greater
-                })
-            };
-            while start_index < end_index {
-                let start_block_table_id = block_metas[start_index].table_id();
-                if self
-                    .existing_table_ids
-                    .contains(&block_metas[start_index].table_id().table_id)
-                {
-                    break;
-                }
-                start_index += &block_metas[(start_index + 1)..]
-                    .partition_point(|block_meta| block_meta.table_id() == start_block_table_id)
-                    + 1;
-            }
-            if start_index >= end_index {
+
+            let block_metas = Self::filter_block_metas(
+                &sstable.meta.block_metas,
+                &self.existing_table_ids,
+                filter_key_range,
+            );
+
+            if block_metas.is_empty() {
                 found = false;
             } else {
                 self.task_progress.inc_num_pending_read_io();
                 let mut sstable_iter = SstableStreamIterator::new(
-                    sstable.meta.block_metas.clone(),
+                    block_metas,
                     table_info.clone(),
                     self.existing_table_ids.clone(),
-                    start_index,
                     &self.stats,
                     self.task_progress.clone(),
                     self.sstable_store.clone(),
@@ -426,6 +409,94 @@ impl ConcatSstableIterator {
         }
         Ok(())
     }
+
+    pub fn filter_block_metas(
+        block_metas: &Vec<BlockMeta>,
+        existing_table_ids: &HashSet<u32>,
+        key_range: KeyRange,
+    ) -> Vec<BlockMeta> {
+        if block_metas.is_empty() {
+            return vec![];
+        }
+
+        let mut start_index = if key_range.left.is_empty() {
+            0
+        } else {
+            // start_index points to the greatest block whose smallest_key <= seek_key.
+            block_metas
+                .partition_point(|block| {
+                    KeyComparator::compare_encoded_full_key(&key_range.left, &block.smallest_key)
+                        != Ordering::Less
+                })
+                .saturating_sub(1)
+        };
+
+        let mut end_index = if key_range.right.is_empty() {
+            block_metas.len()
+        } else {
+            let ret = block_metas.partition_point(|block| {
+                KeyComparator::compare_encoded_full_key(&block.smallest_key, &key_range.right)
+                    != Ordering::Greater
+            });
+
+            if ret == 0 {
+                // not found
+                return vec![];
+            }
+
+            ret
+        }
+        .saturating_sub(1);
+
+        // skip blocks that are not in existing_table_ids
+        while start_index <= end_index {
+            let start_block_table_id = block_metas[start_index].table_id().table_id();
+            if existing_table_ids.contains(&start_block_table_id) {
+                break;
+            }
+
+            // skip this table_id
+            let old_start_index = start_index;
+            let block_metas_to_search = &block_metas[start_index..=end_index];
+
+            start_index += block_metas_to_search.partition_point(|block_meta| {
+                block_meta.table_id().table_id() == start_block_table_id
+            });
+
+            if old_start_index == start_index {
+                // no more blocks with the same table_id
+                break;
+            }
+        }
+
+        while start_index <= end_index {
+            let end_block_table_id = block_metas[end_index].table_id().table_id();
+            if existing_table_ids.contains(&end_block_table_id) {
+                break;
+            }
+
+            let old_end_index = end_index;
+            let block_metas_to_search = &block_metas[start_index..=end_index];
+
+            end_index = start_index
+                + block_metas_to_search
+                    .partition_point(|block_meta| {
+                        block_meta.table_id().table_id() < end_block_table_id
+                    })
+                    .saturating_sub(1);
+
+            if end_index == old_end_index {
+                // no more blocks with the same table_id
+                break;
+            }
+        }
+
+        if start_index > end_index {
+            return vec![];
+        }
+
+        block_metas[start_index..=end_index].to_vec()
+    }
 }
 
 impl HummockIterator for ConcatSstableIterator {
@@ -478,7 +549,7 @@ impl HummockIterator for ConcatSstableIterator {
 
             // Note that we need to use `<` instead of `<=` to ensure that all keys in an SST
             // (including its max. key) produce the same search result.
-            let max_sst_key = &table.key_range.as_ref().unwrap().right;
+            let max_sst_key = &table.key_range.right;
             FullKey::decode(max_sst_key).cmp(&seek_key) == Ordering::Less
         });
 
@@ -491,12 +562,12 @@ impl HummockIterator for ConcatSstableIterator {
 
     fn value_meta(&self) -> ValueMeta {
         let iter = self.sstable_iter.as_ref().expect("no table iter");
-        // sstable_iter's seek_block_idx must have advanced at least one.
+        // sstable_iter's block_idx must have advanced at least one.
         // See SstableStreamIterator::next_block.
-        assert!(iter.seek_block_idx >= 1);
+        assert!(iter.block_idx >= 1);
         ValueMeta {
             object_id: Some(iter.sstable_info.object_id),
-            block_id: Some(iter.seek_block_idx as u64 - 1),
+            block_id: Some(iter.block_idx as u64 - 1),
         }
     }
 }
@@ -569,7 +640,9 @@ impl<I: HummockIterator<Direction = Forward>> HummockIterator for MonitoredCompa
 #[cfg(test)]
 mod tests {
     use std::cmp::Ordering;
+    use std::collections::HashSet;
 
+    use risingwave_common::catalog::TableId;
     use risingwave_hummock_sdk::key::{next_full_key, prev_full_key, FullKey};
     use risingwave_hummock_sdk::key_range::KeyRange;
 
@@ -581,6 +654,7 @@ mod tests {
         TEST_KEYS_COUNT,
     };
     use crate::hummock::value::HummockValue;
+    use crate::hummock::BlockMeta;
 
     #[tokio::test]
     async fn test_concat_iterator() {
@@ -783,9 +857,252 @@ mod tests {
             .unwrap();
         assert!(iter.is_valid());
         assert_eq!(iter.key(), block_1_second_key.to_ref());
+
         // Use None seek key and result in the second KV of block 1.
         iter.seek_idx(0, None).await.unwrap();
         assert!(iter.is_valid());
         assert_eq!(iter.key(), block_1_second_key.to_ref());
     }
+
+    #[tokio::test]
+    async fn test_filter_block_metas() {
+        {
+            let block_metas = Vec::default();
+
+            let ret = ConcatSstableIterator::filter_block_metas(
+                &block_metas,
+                &HashSet::default(),
+                KeyRange::default(),
+            );
+
+            assert!(ret.is_empty());
+        }
+
+        {
+            let block_metas = vec![
+                BlockMeta {
+                    smallest_key: FullKey::for_test(TableId::new(1), Vec::default(), 0).encode(),
+                    ..Default::default()
+                },
+                BlockMeta {
+                    smallest_key: FullKey::for_test(TableId::new(2), Vec::default(), 0).encode(),
+                    ..Default::default()
+                },
+                BlockMeta {
+                    smallest_key: FullKey::for_test(TableId::new(3), Vec::default(), 0).encode(),
+                    ..Default::default()
+                },
+            ];
+
+            let ret = ConcatSstableIterator::filter_block_metas(
+                &block_metas,
+                &HashSet::from_iter(vec![1_u32, 2, 3].into_iter()),
+                KeyRange::default(),
+            );
+
+            assert_eq!(3, ret.len());
+            assert_eq!(
+                1,
+                FullKey::decode(&ret[0].smallest_key)
+                    .user_key
+                    .table_id
+                    .table_id()
+            );
+            assert_eq!(
+                3,
+                FullKey::decode(&ret[2].smallest_key)
+                    .user_key
+                    .table_id
+                    .table_id()
+            );
+        }
+
+        {
+            let block_metas = vec![
+                BlockMeta {
+                    smallest_key: FullKey::for_test(TableId::new(1), Vec::default(), 0).encode(),
+                    ..Default::default()
+                },
+                BlockMeta {
+                    smallest_key: FullKey::for_test(TableId::new(2), Vec::default(), 0).encode(),
+                    ..Default::default()
+                },
+                BlockMeta {
+                    smallest_key: FullKey::for_test(TableId::new(3), Vec::default(), 0).encode(),
+                    ..Default::default()
+                },
+            ];
+
+            let ret = ConcatSstableIterator::filter_block_metas(
+                &block_metas,
+                &HashSet::from_iter(vec![2_u32, 3].into_iter()),
+                KeyRange::default(),
+            );
+
+            assert_eq!(2, ret.len());
+            assert_eq!(
+                2,
+                FullKey::decode(&ret[0].smallest_key)
+                    .user_key
+                    .table_id
+                    .table_id()
+            );
+            assert_eq!(
+                3,
+                FullKey::decode(&ret[1].smallest_key)
+                    .user_key
+                    .table_id
+                    .table_id()
+            );
+        }
+
+        {
+            let block_metas = vec![
+                BlockMeta {
+                    smallest_key: FullKey::for_test(TableId::new(1), Vec::default(), 0).encode(),
+                    ..Default::default()
+                },
+                BlockMeta {
+                    smallest_key: FullKey::for_test(TableId::new(2), Vec::default(), 0).encode(),
+                    ..Default::default()
+                },
+                BlockMeta {
+                    smallest_key: FullKey::for_test(TableId::new(3), Vec::default(), 0).encode(),
+                    ..Default::default()
+                },
+            ];
+
+            let ret = ConcatSstableIterator::filter_block_metas(
+                &block_metas,
+                &HashSet::from_iter(vec![1_u32, 2_u32].into_iter()),
+                KeyRange::default(),
+            );
+
+            assert_eq!(2, ret.len());
+            assert_eq!(
+                1,
+                FullKey::decode(&ret[0].smallest_key)
+                    .user_key
+                    .table_id
+                    .table_id()
+            );
+            assert_eq!(
+                2,
+                FullKey::decode(&ret[1].smallest_key)
+                    .user_key
+                    .table_id
+                    .table_id()
+            );
+        }
+
+        {
+            let block_metas = vec![
+                BlockMeta {
+                    smallest_key: FullKey::for_test(TableId::new(1), Vec::default(), 0).encode(),
+                    ..Default::default()
+                },
+                BlockMeta {
+                    smallest_key: FullKey::for_test(TableId::new(2), Vec::default(), 0).encode(),
+                    ..Default::default()
+                },
+                BlockMeta {
+                    smallest_key: FullKey::for_test(TableId::new(3), Vec::default(), 0).encode(),
+                    ..Default::default()
+                },
+            ];
+            let ret = ConcatSstableIterator::filter_block_metas(
+                &block_metas,
+                &HashSet::from_iter(vec![2_u32].into_iter()),
+                KeyRange::default(),
+            );
+
+            assert_eq!(1, ret.len());
+            assert_eq!(
+                2,
+                FullKey::decode(&ret[0].smallest_key)
+                    .user_key
+                    .table_id
+                    .table_id()
+            );
+        }
+
+        {
+            let block_metas = vec![
+                BlockMeta {
+                    smallest_key: FullKey::for_test(TableId::new(1), Vec::default(), 0).encode(),
+                    ..Default::default()
+                },
+                BlockMeta {
+                    smallest_key: FullKey::for_test(TableId::new(1), Vec::default(), 0).encode(),
+                    ..Default::default()
+                },
+                BlockMeta {
+                    smallest_key: FullKey::for_test(TableId::new(1), Vec::default(), 0).encode(),
+                    ..Default::default()
+                },
+                BlockMeta {
+                    smallest_key: FullKey::for_test(TableId::new(2), Vec::default(), 0).encode(),
+                    ..Default::default()
+                },
+                BlockMeta {
+                    smallest_key: FullKey::for_test(TableId::new(3), Vec::default(), 0).encode(),
+                    ..Default::default()
+                },
+            ];
+            let ret = ConcatSstableIterator::filter_block_metas(
+                &block_metas,
+                &HashSet::from_iter(vec![2_u32].into_iter()),
+                KeyRange::default(),
+            );
+
+            assert_eq!(1, ret.len());
+            assert_eq!(
+                2,
+                FullKey::decode(&ret[0].smallest_key)
+                    .user_key
+                    .table_id
+                    .table_id()
+            );
+        }
+
+        {
+            let block_metas = vec![
+                BlockMeta {
+                    smallest_key: FullKey::for_test(TableId::new(1), Vec::default(), 0).encode(),
+                    ..Default::default()
+                },
+                BlockMeta {
+                    smallest_key: FullKey::for_test(TableId::new(2), Vec::default(), 0).encode(),
+                    ..Default::default()
+                },
+                BlockMeta {
+                    smallest_key: FullKey::for_test(TableId::new(3), Vec::default(), 0).encode(),
+                    ..Default::default()
+                },
+                BlockMeta {
+                    smallest_key: FullKey::for_test(TableId::new(3), Vec::default(), 0).encode(),
+                    ..Default::default()
+                },
+                BlockMeta {
+                    smallest_key: FullKey::for_test(TableId::new(3), Vec::default(), 0).encode(),
+                    ..Default::default()
+                },
+            ];
+
+            let ret = ConcatSstableIterator::filter_block_metas(
+                &block_metas,
+                &HashSet::from_iter(vec![2_u32].into_iter()),
+                KeyRange::default(),
+            );
+
+            assert_eq!(1, ret.len());
+            assert_eq!(
+                2,
+                FullKey::decode(&ret[0].smallest_key)
+                    .user_key
+                    .table_id
+                    .table_id()
+            );
+        }
+    }
 }
diff --git a/src/storage/src/hummock/compactor/mod.rs b/src/storage/src/hummock/compactor/mod.rs
index 898d91d530b22..12e009bb60cb2 100644
--- a/src/storage/src/hummock/compactor/mod.rs
+++ b/src/storage/src/hummock/compactor/mod.rs
@@ -15,12 +15,13 @@
 mod compaction_executor;
 mod compaction_filter;
 pub mod compaction_utils;
+use risingwave_hummock_sdk::compact_task::{CompactTask, ValidationTask};
 use risingwave_pb::compactor::{dispatch_compaction_task_request, DispatchCompactionTaskRequest};
 use risingwave_pb::hummock::report_compaction_task_request::{
     Event as ReportCompactionTaskEvent, HeartBeat as SharedHeartBeat,
     ReportTask as ReportSharedTask,
 };
-use risingwave_pb::hummock::{CompactTask, ReportFullScanTaskRequest, ReportVacuumTaskRequest};
+use risingwave_pb::hummock::{PbCompactTask, ReportFullScanTaskRequest, ReportVacuumTaskRequest};
 use risingwave_rpc_client::GrpcCompactorProxyClient;
 use thiserror_ext::AsReport;
 use tokio::sync::mpsc;
@@ -188,7 +189,7 @@ impl Compactor {
 
         debug_assert!(split_table_outputs
             .iter()
-            .all(|table_info| table_info.sst_info.get_table_ids().is_sorted()));
+            .all(|table_info| table_info.sst_info.table_ids.is_sorted()));
 
         if task_id.is_some() {
             // skip shared buffer compaction
@@ -427,8 +428,12 @@ pub fn start_compactor(
                     if let Err(e) = request_sender.send(SubscribeCompactionEventRequest {
                         event: Some(RequestEvent::ReportTask(ReportTask {
                             task_id: compact_task.task_id,
-                            task_status: compact_task.task_status,
-                            sorted_output_ssts: compact_task.sorted_output_ssts.clone(),
+                            task_status: compact_task.task_status.into(),
+                            sorted_output_ssts: compact_task
+                                .sorted_output_ssts
+                                .iter()
+                                .map(|sst| sst.into())
+                                .collect(),
                             table_stats_change: to_prost_table_stats_map(table_stats),
                         })),
                         create_at: SystemTime::now()
@@ -465,6 +470,7 @@ pub fn start_compactor(
 
                         match event {
                             ResponseEvent::CompactTask(compact_task) => {
+                                let compact_task = CompactTask::from(compact_task);
                                 let parallelism =
                                     calculate_task_parallelism(&compact_task, &context);
 
@@ -531,9 +537,7 @@ pub fn start_compactor(
                                             Err(err) => {
                                                 tracing::warn!(error = %err.as_report(), "Failed to track pending SST object id");
                                                 let mut compact_task = compact_task;
-                                                compact_task.set_task_status(
-                                                    TaskStatus::TrackSstObjectIdFailed,
-                                                );
+                                                compact_task.task_status = TaskStatus::TrackSstObjectIdFailed;
                                                 ((compact_task, HashMap::default()), None)
                                             }
                                         };
@@ -548,7 +552,7 @@ pub fn start_compactor(
 
                                     let enable_check_compaction_result =
                                     context.storage_opts.check_compaction_result;
-                                    let need_check_task = !compact_task.sorted_output_ssts.is_empty() && compact_task.task_status() == TaskStatus::Success;
+                                    let need_check_task = !compact_task.sorted_output_ssts.is_empty() && compact_task.task_status == TaskStatus::Success;
 
                                     if enable_check_compaction_result && need_check_task {
                                         match check_compaction_result(&compact_task, context.clone())
@@ -606,6 +610,7 @@ pub fn start_compactor(
                                 });
                             }
                             ResponseEvent::ValidationTask(validation_task) => {
+                                let validation_task = ValidationTask::from(validation_task);
                                 executor.spawn(async move {
                                     validate_ssts(validation_task, context.sstable_store.clone())
                                         .await;
@@ -729,6 +734,7 @@ pub fn start_shared_compactor(
                             SharedComapctorObjectIdManager::new(output_object_ids_deque, cloned_grpc_proxy_client.clone(), context.storage_opts.sstable_id_remote_fetch_number);
                             match dispatch_task.unwrap() {
                                 dispatch_compaction_task_request::Task::CompactTask(compact_task) => {
+                                    let compact_task = CompactTask::from(&compact_task);
                                     let (tx, rx) = tokio::sync::oneshot::channel();
                                     let task_id = compact_task.task_id;
                                     shutdown.lock().unwrap().insert(task_id, tx);
@@ -744,7 +750,7 @@ pub fn start_shared_compactor(
                                     shutdown.lock().unwrap().remove(&task_id);
                                     let report_compaction_task_request = ReportCompactionTaskRequest {
                                         event: Some(ReportCompactionTaskEvent::ReportTask(ReportSharedTask {
-                                            compact_task: Some(compact_task.clone()),
+                                            compact_task: Some(PbCompactTask::from(&compact_task)),
                                             table_stats_change: to_prost_table_stats_map(table_stats),
                                         })),
                                     };
@@ -756,11 +762,11 @@ pub fn start_shared_compactor(
                                         Ok(_) => {
                                             // TODO: remove this method after we have running risingwave cluster with fast compact algorithm stably for a long time.
                                             let enable_check_compaction_result = context.storage_opts.check_compaction_result;
-                                            let need_check_task =  !compact_task.sorted_output_ssts.is_empty() && compact_task.task_status() == TaskStatus::Success;
+                                            let need_check_task = !compact_task.sorted_output_ssts.is_empty() && compact_task.task_status == TaskStatus::Success;
                                             if enable_check_compaction_result && need_check_task {
                                                 match check_compaction_result(&compact_task, context.clone()).await {
                                                     Err(e) => {
-                                                        tracing::warn!(error = %e.as_report(), "Failed to check compaction task {}", compact_task.task_id);
+                                                        tracing::warn!(error = %e.as_report(), "Failed to check compaction task {}", task_id);
                                                     },
                                                     Ok(true) => (),
                                                     Ok(false) => {
@@ -818,6 +824,7 @@ pub fn start_shared_compactor(
                                     }
                                 }
                                 dispatch_compaction_task_request::Task::ValidationTask(validation_task) => {
+                                    let validation_task = ValidationTask::from(validation_task);
                                     validate_ssts(validation_task, context.sstable_store.clone()).await;
                                 }
                                 dispatch_compaction_task_request::Task::CancelCompactTask(cancel_compact_task) => {
diff --git a/src/storage/src/hummock/compactor/shared_buffer_compact.rs b/src/storage/src/hummock/compactor/shared_buffer_compact.rs
index 52c8cc1710bc2..00680f5906ccf 100644
--- a/src/storage/src/hummock/compactor/shared_buffer_compact.rs
+++ b/src/storage/src/hummock/compactor/shared_buffer_compact.rs
@@ -83,7 +83,6 @@ pub async fn compact(
 
     let mut new_value_futures = vec![];
     for (id, group_payload) in grouped_payload {
-        let id_copy = id;
         new_value_futures.push(
             compact_shared_buffer::<true>(
                 context.clone(),
@@ -91,15 +90,7 @@ pub async fn compact(
                 filter_key_extractor_manager.clone(),
                 group_payload,
             )
-            .map_ok(move |results| {
-                results
-                    .into_iter()
-                    .map(move |mut result| {
-                        result.compaction_group_id = id_copy;
-                        result
-                    })
-                    .collect_vec()
-            })
+            .map_ok(move |results| results.into_iter())
             .instrument_await(format!("shared_buffer_compact_compaction_group {}", id)),
         );
     }
diff --git a/src/storage/src/hummock/event_handler/hummock_event_handler.rs b/src/storage/src/hummock/event_handler/hummock_event_handler.rs
index 4643910b6fd26..cd177e4cf34f3 100644
--- a/src/storage/src/hummock/event_handler/hummock_event_handler.rs
+++ b/src/storage/src/hummock/event_handler/hummock_event_handler.rs
@@ -53,7 +53,6 @@ use crate::hummock::local_version::pinned_version::PinnedVersion;
 use crate::hummock::store::version::{
     HummockReadVersion, StagingData, StagingSstableInfo, VersionUpdate,
 };
-use crate::hummock::utils::validate_table_key_range;
 use crate::hummock::{
     HummockResult, MemoryLimiter, SstableObjectIdManager, SstableStoreRef, TrackerId,
 };
@@ -469,9 +468,9 @@ impl HummockEventHandler {
         table_ids: HashSet<TableId>,
     ) {
         debug!(
-            "awaiting for epoch to be synced: {}, max_synced_epoch: {}",
             new_sync_epoch,
-            self.uploader.max_synced_epoch()
+            ?table_ids,
+            "awaiting for epoch to be synced",
         );
         self.uploader
             .start_sync_epoch(new_sync_epoch, sync_result_sender, table_ids);
@@ -481,7 +480,6 @@ impl HummockEventHandler {
         info!(
             prev_epoch,
             max_committed_epoch = self.uploader.max_committed_epoch(),
-            max_synced_epoch = self.uploader.max_synced_epoch(),
             "handle clear event"
         );
 
@@ -613,8 +611,6 @@ impl HummockEventHandler {
             HummockVersionUpdate::PinnedVersion(version) => *version,
         };
 
-        validate_table_key_range(&newly_pinned_version);
-
         pinned_version.new_pin_version(newly_pinned_version)
     }
 
@@ -738,6 +734,9 @@ impl HummockEventHandler {
             HummockEvent::Shutdown => {
                 unreachable!("shutdown is handled specially")
             }
+            HummockEvent::StartEpoch { epoch, table_ids } => {
+                self.uploader.start_epoch(epoch, table_ids);
+            }
             HummockEvent::InitEpoch {
                 instance_id,
                 init_epoch,
@@ -924,7 +923,7 @@ mod tests {
 
     use futures::FutureExt;
     use parking_lot::Mutex;
-    use risingwave_common::buffer::BitmapBuilder;
+    use risingwave_common::bitmap::BitmapBuilder;
     use risingwave_common::hash::VirtualNode;
     use risingwave_common::util::epoch::{test_epoch, EpochExt};
     use risingwave_hummock_sdk::version::HummockVersion;
@@ -934,7 +933,7 @@ mod tests {
     use tokio::sync::oneshot;
 
     use crate::hummock::event_handler::refiller::CacheRefiller;
-    use crate::hummock::event_handler::uploader::tests::{gen_imm, TEST_TABLE_ID};
+    use crate::hummock::event_handler::uploader::test_utils::{gen_imm, TEST_TABLE_ID};
     use crate::hummock::event_handler::uploader::UploadTaskOutput;
     use crate::hummock::event_handler::{HummockEvent, HummockEventHandler, HummockVersionUpdate};
     use crate::hummock::iterator::test_utils::mock_sstable_store;
@@ -1147,6 +1146,11 @@ mod tests {
             rx.await.unwrap()
         };
 
+        send_event(HummockEvent::StartEpoch {
+            epoch: epoch1,
+            table_ids: HashSet::from_iter([TEST_TABLE_ID]),
+        });
+
         send_event(HummockEvent::InitEpoch {
             instance_id: guard.instance_id,
             init_epoch: epoch1,
@@ -1162,6 +1166,11 @@ mod tests {
             imm: imm1,
         });
 
+        send_event(HummockEvent::StartEpoch {
+            epoch: epoch2,
+            table_ids: HashSet::from_iter([TEST_TABLE_ID]),
+        });
+
         send_event(HummockEvent::LocalSealEpoch {
             instance_id: guard.instance_id,
             next_epoch: epoch2,
@@ -1179,6 +1188,10 @@ mod tests {
         });
 
         let epoch3 = epoch2.next_epoch();
+        send_event(HummockEvent::StartEpoch {
+            epoch: epoch3,
+            table_ids: HashSet::from_iter([TEST_TABLE_ID]),
+        });
         send_event(HummockEvent::LocalSealEpoch {
             instance_id: guard.instance_id,
             next_epoch: epoch3,
diff --git a/src/storage/src/hummock/event_handler/mod.rs b/src/storage/src/hummock/event_handler/mod.rs
index 1d0935c0ac485..39e9d3ecc920a 100644
--- a/src/storage/src/hummock/event_handler/mod.rs
+++ b/src/storage/src/hummock/event_handler/mod.rs
@@ -16,7 +16,7 @@ use std::collections::{HashMap, HashSet};
 use std::sync::Arc;
 
 use parking_lot::{RwLock, RwLockReadGuard};
-use risingwave_common::buffer::Bitmap;
+use risingwave_common::bitmap::Bitmap;
 use risingwave_common::catalog::TableId;
 use risingwave_hummock_sdk::HummockEpoch;
 use thiserror_ext::AsReport;
@@ -74,6 +74,11 @@ pub enum HummockEvent {
         imm: ImmutableMemtable,
     },
 
+    StartEpoch {
+        epoch: HummockEpoch,
+        table_ids: HashSet<TableId>,
+    },
+
     InitEpoch {
         instance_id: LocalInstanceId,
         init_epoch: HummockEpoch,
@@ -117,6 +122,10 @@ impl HummockEvent {
 
             HummockEvent::Shutdown => "Shutdown".to_string(),
 
+            HummockEvent::StartEpoch { epoch, table_ids } => {
+                format!("StartEpoch {} {:?}", epoch, table_ids)
+            }
+
             HummockEvent::InitEpoch {
                 instance_id,
                 init_epoch,
@@ -208,7 +217,7 @@ impl Drop for LocalInstanceGuard {
                     instance_id: self.instance_id,
                 })
                 .unwrap_or_else(|err| {
-                    tracing::error!(
+                    tracing::debug!(
                         error = %err.as_report(),
                         table_id = %self.table_id,
                         instance_id = self.instance_id,
diff --git a/src/storage/src/hummock/event_handler/refiller.rs b/src/storage/src/hummock/event_handler/refiller.rs
index 10ed4484917b6..09526f6b83c17 100644
--- a/src/storage/src/hummock/event_handler/refiller.rs
+++ b/src/storage/src/hummock/event_handler/refiller.rs
@@ -372,6 +372,7 @@ impl CacheRefillTask {
 
                 let now = Instant::now();
                 let res = context.sstable_store.sstable(info, &mut stats).await;
+                stats.discard();
                 GLOBAL_CACHE_REFILL_METRICS
                     .meta_refill_success_duration
                     .observe(now.elapsed().as_secs_f64());
diff --git a/src/storage/src/hummock/event_handler/uploader.rs b/src/storage/src/hummock/event_handler/uploader/mod.rs
similarity index 66%
rename from src/storage/src/hummock/event_handler/uploader.rs
rename to src/storage/src/hummock/event_handler/uploader/mod.rs
index f341937d8795f..8210a998974c4 100644
--- a/src/storage/src/hummock/event_handler/uploader.rs
+++ b/src/storage/src/hummock/event_handler/uploader/mod.rs
@@ -12,13 +12,16 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+mod spiller;
+mod task_manager;
+pub(crate) mod test_utils;
+
 use std::cmp::Ordering;
 use std::collections::btree_map::Entry;
-use std::collections::{BTreeMap, HashMap, HashSet, VecDeque};
+use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque};
 use std::fmt::{Debug, Display, Formatter};
 use std::future::{poll_fn, Future};
 use std::mem::{replace, swap, take};
-use std::pin::Pin;
 use std::sync::Arc;
 use std::task::{ready, Context, Poll};
 
@@ -27,7 +30,7 @@ use itertools::Itertools;
 use more_asserts::{assert_ge, assert_gt};
 use prometheus::core::{AtomicU64, GenericGauge};
 use prometheus::{HistogramTimer, IntGauge};
-use risingwave_common::buffer::BitmapBuilder;
+use risingwave_common::bitmap::BitmapBuilder;
 use risingwave_common::catalog::TableId;
 use risingwave_common::hash::VirtualNode;
 use risingwave_common::must_match;
@@ -35,12 +38,14 @@ use risingwave_hummock_sdk::table_watermark::{
     TableWatermarks, VnodeWatermark, WatermarkDirection,
 };
 use risingwave_hummock_sdk::{CompactionGroupId, HummockEpoch, LocalSstableInfo};
+use task_manager::{TaskManager, UploadingTaskStatus};
 use thiserror_ext::AsReport;
 use tokio::sync::oneshot;
 use tokio::task::JoinHandle;
 use tracing::{debug, error, info, warn};
 
 use crate::hummock::event_handler::hummock_event_handler::{send_sync_result, BufferTracker};
+use crate::hummock::event_handler::uploader::spiller::Spiller;
 use crate::hummock::event_handler::uploader::uploader_imm::UploaderImm;
 use crate::hummock::event_handler::LocalInstanceId;
 use crate::hummock::local_version::pinned_version::PinnedVersion;
@@ -161,9 +166,13 @@ mod uploader_imm {
     }
 }
 
+#[derive(PartialEq, Eq, Hash, PartialOrd, Ord, Copy, Clone, Debug)]
+struct UploadingTaskId(usize);
+
 /// A wrapper for a uploading task that compacts and uploads the imm payload. Task context are
 /// stored so that when the task fails, it can be re-tried.
 struct UploadingTask {
+    task_id: UploadingTaskId,
     // newer data at the front
     input: UploadTaskInput,
     join_handle: JoinHandle<HummockResult<UploadTaskOutput>>,
@@ -219,7 +228,7 @@ impl UploadingTask {
             .collect()
     }
 
-    fn new(input: UploadTaskInput, context: &UploaderContext) -> Self {
+    fn new(task_id: UploadingTaskId, input: UploadTaskInput, context: &UploaderContext) -> Self {
         assert!(!input.is_empty());
         let mut epochs = input
             .iter()
@@ -254,6 +263,7 @@ impl UploadingTask {
         let join_handle = (context.spawn_upload_task)(payload, task_info.clone());
         context.stats.uploader_uploading_task_count.inc();
         Self {
+            task_id,
             input,
             join_handle,
             task_info,
@@ -324,81 +334,6 @@ impl UploadingTask {
     }
 }
 
-impl Future for UploadingTask {
-    type Output = HummockResult<Arc<StagingSstableInfo>>;
-
-    fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
-        self.poll_result(cx)
-    }
-}
-
-#[derive(Default, Debug)]
-/// Manage the spilled data. Task and uploaded data at the front is newer data. Task data are
-/// always newer than uploaded data. Task holding oldest data is always collected first.
-struct SpilledData {
-    // ordered spilling tasks. Task at the back is spilling older data.
-    uploading_tasks: VecDeque<UploadingTask>,
-    // ordered spilled data. Data at the back is older.
-    uploaded_data: VecDeque<Arc<StagingSstableInfo>>,
-}
-
-impl SpilledData {
-    fn add_task(&mut self, task: UploadingTask) {
-        self.uploading_tasks.push_front(task);
-    }
-
-    /// Poll the successful spill of the oldest uploading task. Return `Poll::Ready(None)` is there
-    /// is no uploading task
-    fn poll_success_spill(
-        &mut self,
-        cx: &mut Context<'_>,
-    ) -> Poll<Option<Arc<StagingSstableInfo>>> {
-        // only poll the oldest uploading task if there is any
-        if let Some(task) = self.uploading_tasks.back_mut() {
-            let staging_sstable_info = ready!(task.poll_ok_with_retry(cx));
-            self.uploaded_data.push_front(staging_sstable_info.clone());
-            self.uploading_tasks.pop_back();
-            Poll::Ready(Some(staging_sstable_info))
-        } else {
-            Poll::Ready(None)
-        }
-    }
-
-    fn abort(self) {
-        for task in self.uploading_tasks {
-            task.join_handle.abort();
-        }
-    }
-}
-
-#[derive(Default, Debug)]
-struct EpochData {
-    spilled_data: SpilledData,
-}
-
-impl EpochData {
-    fn flush(
-        &mut self,
-        context: &UploaderContext,
-        imms: HashMap<LocalInstanceId, Vec<UploaderImm>>,
-    ) -> usize {
-        if !imms.is_empty() {
-            let task = UploadingTask::new(imms, context);
-            context.stats.spill_task_counts_from_unsealed.inc();
-            context
-                .stats
-                .spill_task_size_from_unsealed
-                .inc_by(task.task_info.task_size as u64);
-            info!("Spill unsealed data. Task: {}", task.get_task_info());
-            let size = task.task_info.task_size;
-            self.spilled_data.add_task(task);
-            size
-        } else {
-            0
-        }
-    }
-}
-
 impl TableUnsyncData {
     fn add_table_watermarks(
         &mut self,
@@ -453,70 +388,9 @@ impl TableUnsyncData {
     }
 }
 
-#[derive(Default)]
-struct SyncDataBuilder {
-    // newer epochs come first
-    epochs: VecDeque<HummockEpoch>,
-
-    spilled_data: SpilledData,
-
-    table_watermarks: HashMap<TableId, TableWatermarks>,
-}
-
-impl SyncDataBuilder {
-    /// Add the data of a new epoch.
-    ///
-    /// Note: it may happen that, for example, currently we hold `imms` and `spilled_data` of epoch
-    /// 3,  and after we add the spilled data of epoch 4, both `imms` and `spilled_data` hold data
-    /// of both epoch 3 and 4, which seems breaking the rules that data in `imms` are
-    /// always newer than data in `spilled_data`, because epoch 3 data of `imms`
-    /// seems older than epoch 4 data of `spilled_data`. However, if this happens, the epoch 3
-    /// data of `imms` must not overlap with the epoch 4 data of `spilled_data`. The explanation is
-    /// as followed:
-    ///
-    /// First, unsync data has 3 stages, from earlier to later, imms, uploading task, and
-    /// uploaded. When we try to spill unsync data, we first pick the imms of older epoch until
-    /// the imms of older epoch are all picked. When we try to poll the uploading tasks of unsync
-    /// data, we first poll the task of older epoch, until there is no uploading task in older
-    /// epoch. Therefore, we can reach that, if two data are in the same stage, but
-    /// different epochs, data in the older epoch will always enter the next stage earlier than data
-    /// in the newer epoch.
-    ///
-    /// Second, we have an assumption that, if a key has been written in a newer epoch, e.g. epoch4,
-    /// it will no longer be written in an older epoch, e.g. epoch3, and then, if two data of the
-    /// same key are at the imm stage, the data of older epoch must appear earlier than the data
-    /// of newer epoch.
-    ///
-    /// Based on the two points above, we can reach that, if two data of a same key appear in
-    /// different epochs, the data of older epoch will not appear at a later stage than the data
-    /// of newer epoch. Therefore, we can safely merge the data of each stage when we seal an epoch.
-    fn add_new_epoch(&mut self, epoch: HummockEpoch, mut unseal_epoch_data: EpochData) {
-        if let Some(prev_max_epoch) = self.epochs.front() {
-            assert!(
-                epoch > *prev_max_epoch,
-                "epoch {} to seal not greater than prev max epoch {}",
-                epoch,
-                prev_max_epoch
-            );
-        }
-
-        self.epochs.push_front(epoch);
-        // for each local instance, earlier data must be spilled at earlier epoch. Therefore, since we add spill data from old epoch
-        // to new epoch,
-        unseal_epoch_data
-            .spilled_data
-            .uploading_tasks
-            .append(&mut self.spilled_data.uploading_tasks);
-        unseal_epoch_data
-            .spilled_data
-            .uploaded_data
-            .append(&mut self.spilled_data.uploaded_data);
-        self.spilled_data.uploading_tasks = unseal_epoch_data.spilled_data.uploading_tasks;
-        self.spilled_data.uploaded_data = unseal_epoch_data.spilled_data.uploaded_data;
-    }
-
+impl UploaderData {
     fn add_table_watermarks(
-        &mut self,
+        all_table_watermarks: &mut HashMap<TableId, TableWatermarks>,
         table_id: TableId,
         direction: WatermarkDirection,
         watermarks: impl Iterator<Item = (HummockEpoch, Vec<VnodeWatermark>)>,
@@ -538,19 +412,11 @@ impl SyncDataBuilder {
             }
         }
         if let Some(table_watermarks) = table_watermarks {
-            assert!(self
-                .table_watermarks
+            assert!(all_table_watermarks
                 .insert(table_id, table_watermarks)
                 .is_none());
         }
     }
-
-    fn flush(&mut self, context: &UploaderContext, payload: UploadTaskInput) {
-        if !payload.is_empty() {
-            let task = UploadingTask::new(payload, context);
-            self.spilled_data.add_task(task);
-        }
-    }
 }
 
 struct LocalInstanceEpochData {
@@ -718,7 +584,7 @@ impl LocalInstanceUnsyncData {
                 assert!(latest_epoch_data.is_empty());
                 assert!(!latest_epoch_data.has_spilled);
                 if cfg!(debug_assertions) {
-                    panic!("sync epoch exceeds latest epoch, and the current instance should have be archived");
+                    panic!("sync epoch exceeds latest epoch, and the current instance should have been archived");
                 }
                 warn!(
                     instance_id = self.instance_id,
@@ -751,15 +617,37 @@ struct TableUnsyncData {
         WatermarkDirection,
         BTreeMap<HummockEpoch, (Vec<VnodeWatermark>, BitmapBuilder)>,
     )>,
+    spill_tasks: BTreeMap<HummockEpoch, VecDeque<UploadingTaskId>>,
+    unsync_epochs: BTreeMap<HummockEpoch, ()>,
+    // Initialized to be `None`. Transform to `Some(_)` when called
+    // `local_seal_epoch` with a non-existing epoch, to mark that
+    // the fragment of the table has stopped.
+    stopped_next_epoch: Option<HummockEpoch>,
+    // newer epoch at the front
+    syncing_epochs: VecDeque<HummockEpoch>,
+    max_synced_epoch: Option<HummockEpoch>,
 }
 
 impl TableUnsyncData {
-    fn new(table_id: TableId) -> Self {
+    fn new(table_id: TableId, committed_epoch: Option<HummockEpoch>) -> Self {
         Self {
             table_id,
             instance_data: Default::default(),
             table_watermarks: None,
+            spill_tasks: Default::default(),
+            unsync_epochs: Default::default(),
+            stopped_next_epoch: None,
+            syncing_epochs: Default::default(),
+            max_synced_epoch: committed_epoch,
+        }
+    }
+
+    fn new_epoch(&mut self, epoch: HummockEpoch) {
+        debug!(table_id = ?self.table_id, epoch, "table new epoch");
+        if let Some(latest_epoch) = self.max_epoch() {
+            assert_gt!(epoch, latest_epoch);
         }
+        self.unsync_epochs.insert(epoch, ());
     }
 
     fn sync(
@@ -771,7 +659,20 @@ impl TableUnsyncData {
             WatermarkDirection,
             impl Iterator<Item = (HummockEpoch, Vec<VnodeWatermark>)>,
         )>,
+        impl Iterator<Item = UploadingTaskId>,
+        BTreeMap<HummockEpoch, ()>,
     ) {
+        if let Some(prev_epoch) = self.max_sync_epoch() {
+            assert_gt!(epoch, prev_epoch)
+        }
+        let epochs = take_before_epoch(&mut self.unsync_epochs, epoch);
+        assert_eq!(
+            *epochs.last_key_value().expect("non-empty").0,
+            epoch,
+            "{epochs:?} {epoch} {:?}",
+            self.table_id
+        );
+        self.syncing_epochs.push_front(epoch);
         (
             self.instance_data
                 .iter_mut()
@@ -787,9 +688,38 @@ impl TableUnsyncData {
                             .map(|(epoch, (watermarks, _))| (epoch, watermarks)),
                     )
                 }),
+            take_before_epoch(&mut self.spill_tasks, epoch)
+                .into_values()
+                .flat_map(|tasks| tasks.into_iter()),
+            epochs,
         )
     }
 
+    fn ack_synced(&mut self, sync_epoch: HummockEpoch) {
+        let min_sync_epoch = self.syncing_epochs.pop_back().expect("should exist");
+        assert_eq!(sync_epoch, min_sync_epoch);
+        self.max_synced_epoch = Some(sync_epoch);
+    }
+
+    fn ack_committed(&mut self, committed_epoch: HummockEpoch) {
+        let synced_epoch_advanced = {
+            if let Some(max_synced_epoch) = self.max_synced_epoch
+                && max_synced_epoch >= committed_epoch
+            {
+                false
+            } else {
+                true
+            }
+        };
+        if synced_epoch_advanced {
+            self.max_synced_epoch = Some(committed_epoch);
+            if let Some(min_syncing_epoch) = self.syncing_epochs.back() {
+                assert_gt!(*min_syncing_epoch, committed_epoch);
+            }
+            self.assert_after_epoch(committed_epoch);
+        }
+    }
+
     fn assert_after_epoch(&self, epoch: HummockEpoch) {
         self.instance_data
             .values()
@@ -800,6 +730,42 @@ impl TableUnsyncData {
             assert_gt!(*oldest_epoch, epoch);
         }
     }
+
+    fn max_sync_epoch(&self) -> Option<HummockEpoch> {
+        self.syncing_epochs
+            .front()
+            .cloned()
+            .or(self.max_synced_epoch)
+    }
+
+    fn max_epoch(&self) -> Option<HummockEpoch> {
+        self.unsync_epochs
+            .last_key_value()
+            .map(|(epoch, _)| *epoch)
+            .or_else(|| self.max_sync_epoch())
+    }
+
+    fn is_empty(&self) -> bool {
+        self.instance_data.is_empty()
+            && self.syncing_epochs.is_empty()
+            && self.unsync_epochs.is_empty()
+    }
+}
+
+#[derive(Eq, Hash, PartialEq, Copy, Clone)]
+struct UnsyncEpochId(HummockEpoch, TableId);
+
+impl UnsyncEpochId {
+    fn epoch(&self) -> HummockEpoch {
+        self.0
+    }
+}
+
+fn get_unsync_epoch_id(epoch: HummockEpoch, table_ids: &HashSet<TableId>) -> Option<UnsyncEpochId> {
+    table_ids
+        .iter()
+        .min()
+        .map(|table_id| UnsyncEpochId(epoch, *table_id))
 }
 
 #[derive(Default)]
@@ -811,7 +777,7 @@ struct UnsyncData {
     table_data: HashMap<TableId, TableUnsyncData>,
     // An index as a mapping from instance id to its table id
     instance_table_id: HashMap<LocalInstanceId, TableId>,
-    epoch_data: BTreeMap<HummockEpoch, EpochData>,
+    unsync_epochs: HashMap<UnsyncEpochId, HashSet<TableId>>,
 }
 
 impl UnsyncData {
@@ -827,8 +793,8 @@ impl UnsyncData {
         );
         let table_data = self
             .table_data
-            .entry(table_id)
-            .or_insert_with(|| TableUnsyncData::new(table_id));
+            .get_mut(&table_id)
+            .unwrap_or_else(|| panic!("should exist. {table_id:?}"));
         assert!(table_data
             .instance_data
             .insert(
@@ -840,7 +806,7 @@ impl UnsyncData {
             .instance_table_id
             .insert(instance_id, table_id)
             .is_none());
-        self.epoch_data.entry(init_epoch).or_default();
+        assert!(table_data.unsync_epochs.contains_key(&init_epoch));
     }
 
     fn instance_data(
@@ -879,56 +845,172 @@ impl UnsyncData {
             .get_mut(&instance_id)
             .expect("should exist");
         let epoch = instance_data.local_seal_epoch(next_epoch);
-        self.epoch_data.entry(next_epoch).or_default();
+        // When drop/cancel a streaming job, for the barrier to stop actor, the
+        // local instance will call `local_seal_epoch`, but the `next_epoch` won't be
+        // called `start_epoch` because we have stopped writing on it.
+        if !table_data.unsync_epochs.contains_key(&next_epoch) {
+            if let Some(stopped_next_epoch) = table_data.stopped_next_epoch {
+                if stopped_next_epoch != next_epoch {
+                    let table_id = table_data.table_id.table_id;
+                    let unsync_epochs = table_data.unsync_epochs.keys().collect_vec();
+                    if cfg!(debug_assertions) {
+                        panic!(
+                            "table_id {} stop epoch {} different to prev stop epoch {}. unsync epochs: {:?}, syncing epochs {:?}, max_synced_epoch {:?}",
+                            table_id, next_epoch, stopped_next_epoch, unsync_epochs, table_data.syncing_epochs, table_data.max_synced_epoch
+                        );
+                    } else {
+                        warn!(
+                            table_id,
+                            stopped_next_epoch,
+                            next_epoch,
+                            ?unsync_epochs,
+                            syncing_epochs = ?table_data.syncing_epochs,
+                            max_synced_epoch = ?table_data.max_synced_epoch,
+                            "different stop epoch"
+                        );
+                    }
+                }
+            } else {
+                if let Some(max_epoch) = table_data.max_epoch() {
+                    assert_gt!(next_epoch, max_epoch);
+                }
+                debug!(?table_id, epoch, next_epoch, "table data has stopped");
+                table_data.stopped_next_epoch = Some(next_epoch);
+            }
+        }
         if let Some((direction, table_watermarks)) = opts.table_watermarks {
             table_data.add_table_watermarks(epoch, table_watermarks, direction);
         }
     }
 
-    fn may_destroy_instance(&mut self, instance_id: LocalInstanceId) {
+    fn may_destroy_instance(&mut self, instance_id: LocalInstanceId) -> Option<TableUnsyncData> {
         if let Some(table_id) = self.instance_table_id.remove(&instance_id) {
             debug!(instance_id, "destroy instance");
             let table_data = self.table_data.get_mut(&table_id).expect("should exist");
             assert!(table_data.instance_data.remove(&instance_id).is_some());
-            if table_data.instance_data.is_empty() {
-                self.table_data.remove(&table_id);
+            if table_data.is_empty() {
+                Some(self.table_data.remove(&table_id).expect("should exist"))
+            } else {
+                None
             }
+        } else {
+            None
         }
     }
+}
 
+impl UploaderData {
     fn sync(
         &mut self,
         epoch: HummockEpoch,
         context: &UploaderContext,
         table_ids: HashSet<TableId>,
-    ) -> SyncDataBuilder {
-        let sync_epoch_data = take_before_epoch(&mut self.epoch_data, epoch);
-
-        let mut sync_data = SyncDataBuilder::default();
-        for (epoch, epoch_data) in sync_epoch_data {
-            sync_data.add_new_epoch(epoch, epoch_data);
-        }
+        sync_result_sender: oneshot::Sender<HummockResult<SyncedData>>,
+    ) {
+        let mut all_table_watermarks = HashMap::new();
+        let mut uploading_tasks = HashSet::new();
+        let mut spilled_tasks = BTreeSet::new();
 
         let mut flush_payload = HashMap::new();
-        for (table_id, table_data) in &mut self.table_data {
-            if !table_ids.contains(table_id) {
-                table_data.assert_after_epoch(epoch);
-                continue;
+
+        if let Some(UnsyncEpochId(_, min_table_id)) = get_unsync_epoch_id(epoch, &table_ids) {
+            let min_table_id_data = self
+                .unsync_data
+                .table_data
+                .get_mut(&min_table_id)
+                .expect("should exist");
+            let epochs = take_before_epoch(&mut min_table_id_data.unsync_epochs.clone(), epoch);
+            for epoch in epochs.keys() {
+                assert_eq!(
+                    self.unsync_data
+                        .unsync_epochs
+                        .remove(&UnsyncEpochId(*epoch, min_table_id))
+                        .expect("should exist"),
+                    table_ids
+                );
             }
-            let (unflushed_payload, table_watermarks) = table_data.sync(epoch);
-            for (instance_id, payload) in unflushed_payload {
-                if !payload.is_empty() {
-                    flush_payload.insert(instance_id, payload);
+            for table_id in &table_ids {
+                let table_data = self
+                    .unsync_data
+                    .table_data
+                    .get_mut(table_id)
+                    .expect("should exist");
+                let (unflushed_payload, table_watermarks, task_ids, table_unsync_epochs) =
+                    table_data.sync(epoch);
+                assert_eq!(table_unsync_epochs, epochs);
+                for (instance_id, payload) in unflushed_payload {
+                    if !payload.is_empty() {
+                        flush_payload.insert(instance_id, payload);
+                    }
+                }
+                if let Some((direction, watermarks)) = table_watermarks {
+                    Self::add_table_watermarks(
+                        &mut all_table_watermarks,
+                        *table_id,
+                        direction,
+                        watermarks,
+                    );
+                }
+                for task_id in task_ids {
+                    if self.spilled_data.contains_key(&task_id) {
+                        spilled_tasks.insert(task_id);
+                    } else {
+                        uploading_tasks.insert(task_id);
+                    }
                 }
             }
-            if let Some((direction, watermarks)) = table_watermarks {
-                sync_data.add_table_watermarks(*table_id, direction, watermarks);
-            }
         }
-        sync_data.flush(context, flush_payload);
-        sync_data
+
+        let sync_id = {
+            let sync_id = self.next_sync_id;
+            self.next_sync_id += 1;
+            SyncId(sync_id)
+        };
+
+        if let Some(extra_flush_task_id) = self.task_manager.sync(
+            context,
+            sync_id,
+            flush_payload,
+            uploading_tasks.iter().cloned(),
+            &table_ids,
+        ) {
+            uploading_tasks.insert(extra_flush_task_id);
+        }
+
+        // iter from large task_id to small one so that newer data at the front
+        let uploaded = spilled_tasks
+            .iter()
+            .rev()
+            .map(|task_id| {
+                let (sst, spill_table_ids) =
+                    self.spilled_data.remove(task_id).expect("should exist");
+                assert!(
+                    spill_table_ids.is_subset(&table_ids),
+                    "spilled tabled ids {:?} not a subset of sync table id {:?}",
+                    spill_table_ids,
+                    table_ids
+                );
+                sst
+            })
+            .collect();
+
+        self.syncing_data.insert(
+            sync_id,
+            SyncingData {
+                sync_epoch: epoch,
+                table_ids,
+                remaining_uploading_tasks: uploading_tasks,
+                uploaded,
+                table_watermarks: all_table_watermarks,
+                sync_result_sender,
+            },
+        );
+
+        self.check_upload_task_consistency();
     }
+}
 
+impl UnsyncData {
     fn ack_flushed(&mut self, sstable_info: &StagingSstableInfo) {
         for (instance_id, imm_ids) in sstable_info.imm_ids() {
             if let Some(instance_data) = self.instance_data(*instance_id) {
@@ -941,8 +1023,8 @@ impl UnsyncData {
 
 struct SyncingData {
     sync_epoch: HummockEpoch,
-    // task of newer data at the front
-    uploading_tasks: VecDeque<UploadingTask>,
+    table_ids: HashSet<TableId>,
+    remaining_uploading_tasks: HashSet<UploadingTaskId>,
     // newer data at the front
     uploaded: VecDeque<Arc<StagingSstableInfo>>,
     table_watermarks: HashMap<TableId, TableWatermarks>,
@@ -981,25 +1063,24 @@ impl UploaderContext {
     }
 }
 
+#[derive(PartialEq, Eq, Hash, PartialOrd, Ord, Copy, Clone, Debug)]
+struct SyncId(usize);
+
 #[derive(Default)]
 struct UploaderData {
     unsync_data: UnsyncData,
 
-    /// Data that has started syncing but not synced yet. `epoch` satisfies
-    /// `max_synced_epoch < epoch <= max_syncing_epoch`.
-    /// Newer epoch at the front
-    syncing_data: VecDeque<SyncingData>,
+    syncing_data: BTreeMap<SyncId, SyncingData>,
+
+    task_manager: TaskManager,
+    spilled_data: HashMap<UploadingTaskId, (Arc<StagingSstableInfo>, HashSet<TableId>)>,
+    next_sync_id: usize,
 }
 
 impl UploaderData {
     fn abort(self, err: impl Fn() -> HummockError) {
-        for (_, epoch_data) in self.unsync_data.epoch_data {
-            epoch_data.spilled_data.abort();
-        }
-        for syncing_data in self.syncing_data {
-            for task in syncing_data.uploading_tasks {
-                task.join_handle.abort();
-            }
+        self.task_manager.abort();
+        for syncing_data in self.syncing_data.into_values() {
             send_sync_result(syncing_data.sync_result_sender, Err(err()));
         }
     }
@@ -1020,7 +1101,7 @@ enum UploaderState {
 /// Data have 3 sequential stages: unsync (inside each local instance, data can be unsealed, sealed), syncing, synced.
 ///
 /// The 3 stages are divided by 2 marginal epochs: `max_syncing_epoch`,
-/// `max_synced_epoch`. Epochs satisfy the following inequality.
+/// `max_synced_epoch` in each `TableUnSyncData`. Epochs satisfy the following inequality.
 ///
 /// (epochs of `synced_data`) <= `max_synced_epoch` < (epochs of `syncing_data`) <=
 /// `max_syncing_epoch` < (epochs of `unsync_data`)
@@ -1028,11 +1109,6 @@ enum UploaderState {
 /// Data are mostly stored in `VecDeque`, and the order stored in the `VecDeque` indicates the data
 /// order. Data at the front represents ***newer*** data.
 pub struct HummockUploader {
-    /// The maximum epoch that has started syncing
-    max_syncing_epoch: HummockEpoch,
-    /// The maximum epoch that has been synced
-    max_synced_epoch: HummockEpoch,
-
     state: UploaderState,
 
     context: UploaderContext,
@@ -1046,14 +1122,8 @@ impl HummockUploader {
         buffer_tracker: BufferTracker,
         config: &StorageOpts,
     ) -> Self {
-        let initial_epoch = pinned_version.version().max_committed_epoch;
         Self {
-            max_syncing_epoch: initial_epoch,
-            max_synced_epoch: initial_epoch,
-            state: UploaderState::Working(UploaderData {
-                unsync_data: Default::default(),
-                syncing_data: Default::default(),
-            }),
+            state: UploaderState::Working(UploaderData::default()),
             context: UploaderContext::new(
                 pinned_version,
                 spawn_upload_task,
@@ -1068,10 +1138,6 @@ impl HummockUploader {
         &self.context.buffer_tracker
     }
 
-    pub(super) fn max_synced_epoch(&self) -> HummockEpoch {
-        self.max_synced_epoch
-    }
-
     pub(super) fn max_committed_epoch(&self) -> HummockEpoch {
         self.context.pinned_version.max_committed_epoch()
     }
@@ -1097,7 +1163,6 @@ impl HummockUploader {
         let UploaderState::Working(data) = &mut self.state else {
             return;
         };
-        assert_gt!(init_epoch, self.max_syncing_epoch);
         data.unsync_data
             .init_instance(table_id, instance_id, init_epoch);
     }
@@ -1111,11 +1176,43 @@ impl HummockUploader {
         let UploaderState::Working(data) = &mut self.state else {
             return;
         };
-        assert_gt!(next_epoch, self.max_syncing_epoch);
         data.unsync_data
             .local_seal_epoch(instance_id, next_epoch, opts);
     }
 
+    pub(super) fn start_epoch(&mut self, epoch: HummockEpoch, table_ids: HashSet<TableId>) {
+        let UploaderState::Working(data) = &mut self.state else {
+            return;
+        };
+        debug!(epoch, ?table_ids, "start epoch");
+        for table_id in &table_ids {
+            let table_data = data
+                .unsync_data
+                .table_data
+                .entry(*table_id)
+                .or_insert_with(|| {
+                    TableUnsyncData::new(
+                        *table_id,
+                        self.context
+                            .pinned_version
+                            .version()
+                            .state_table_info
+                            .info()
+                            .get(table_id)
+                            .map(|info| info.committed_epoch),
+                    )
+                });
+            table_data.new_epoch(epoch);
+        }
+        if let Some(unsync_epoch_id) = get_unsync_epoch_id(epoch, &table_ids) {
+            assert!(data
+                .unsync_data
+                .unsync_epochs
+                .insert(unsync_epoch_id, table_ids)
+                .is_none());
+        }
+    }
+
     pub(super) fn start_sync_epoch(
         &mut self,
         epoch: HummockEpoch,
@@ -1136,35 +1233,11 @@ impl HummockUploader {
                 return;
             }
         };
-        debug!("start sync epoch: {}", epoch);
-        assert!(
-            epoch > self.max_syncing_epoch,
-            "the epoch {} has started syncing already: {}",
-            epoch,
-            self.max_syncing_epoch
-        );
+        debug!(epoch, ?table_ids, "start sync epoch");
 
-        self.max_syncing_epoch = epoch;
-
-        let sync_data = data.unsync_data.sync(epoch, &self.context, table_ids);
-
-        let SyncDataBuilder {
-            spilled_data:
-                SpilledData {
-                    uploading_tasks,
-                    uploaded_data,
-                },
-            table_watermarks,
-            ..
-        } = sync_data;
-
-        data.syncing_data.push_front(SyncingData {
-            sync_epoch: epoch,
-            uploading_tasks,
-            uploaded: uploaded_data,
-            table_watermarks,
-            sync_result_sender,
-        });
+        data.sync(epoch, &self.context, table_ids, sync_result_sender);
+
+        data.may_notify_sync_task(&self.context);
 
         self.context
             .stats
@@ -1172,53 +1245,20 @@ impl HummockUploader {
             .set(data.syncing_data.len() as _);
     }
 
-    fn set_max_synced_epoch(
-        max_synced_epoch: &mut HummockEpoch,
-        max_syncing_epoch: HummockEpoch,
-        epoch: HummockEpoch,
-    ) {
-        assert!(
-            epoch <= max_syncing_epoch,
-            "epoch {} that has been synced has not started syncing yet.  previous max syncing epoch {}",
-            epoch,
-            max_syncing_epoch
-        );
-        assert!(
-            epoch > *max_synced_epoch,
-            "epoch {} has been synced. previous max synced epoch: {}",
-            epoch,
-            max_synced_epoch
-        );
-        *max_synced_epoch = epoch;
-    }
-
     pub(crate) fn update_pinned_version(&mut self, pinned_version: PinnedVersion) {
         assert_ge!(
             pinned_version.max_committed_epoch(),
             self.context.pinned_version.max_committed_epoch()
         );
-        let max_committed_epoch = pinned_version.max_committed_epoch();
-        self.context.pinned_version = pinned_version;
-        if self.max_synced_epoch < max_committed_epoch {
-            self.max_synced_epoch = max_committed_epoch;
-        }
-        if self.max_syncing_epoch < max_committed_epoch {
-            self.max_syncing_epoch = max_committed_epoch;
-            if let UploaderState::Working(data) = &self.state {
-                for instance_data in data
-                    .unsync_data
-                    .table_data
-                    .values()
-                    .flat_map(|data| data.instance_data.values())
-                {
-                    if let Some(oldest_epoch) = instance_data.sealed_data.back() {
-                        assert_gt!(oldest_epoch.epoch, max_committed_epoch);
-                    } else if let Some(current_epoch) = &instance_data.current_epoch_data {
-                        assert_gt!(current_epoch.epoch, max_committed_epoch);
-                    }
+        if let UploaderState::Working(data) = &mut self.state {
+            // TODO: may only `ack_committed` on table whose `committed_epoch` is changed.
+            for (table_id, info) in pinned_version.version().state_table_info.info() {
+                if let Some(table_data) = data.unsync_data.table_data.get_mut(table_id) {
+                    table_data.ack_committed(info.committed_epoch);
                 }
             }
         }
+        self.context.pinned_version = pinned_version;
     }
 
     pub(crate) fn may_flush(&mut self) -> bool {
@@ -1226,30 +1266,35 @@ impl HummockUploader {
             return false;
         };
         if self.context.buffer_tracker.need_flush() {
+            let mut spiller = Spiller::new(&mut data.unsync_data);
             let mut curr_batch_flush_size = 0;
             // iterate from older epoch to newer epoch
-            for (epoch, epoch_data) in &mut data.unsync_data.epoch_data {
-                if !self
-                    .context
-                    .buffer_tracker
-                    .need_more_flush(curr_batch_flush_size)
-                {
-                    break;
-                }
-                let mut payload = HashMap::new();
-                for (instance_id, instance_data) in data
-                    .unsync_data
-                    .table_data
-                    .values_mut()
-                    .flat_map(|data| data.instance_data.iter_mut())
+            while self
+                .context
+                .buffer_tracker
+                .need_more_flush(curr_batch_flush_size)
+                && let Some((epoch, payload, spilled_table_ids)) = spiller.next_spilled_payload()
+            {
+                assert!(!payload.is_empty());
                 {
-                    let instance_payload = instance_data.spill(*epoch);
-                    if !instance_payload.is_empty() {
-                        payload.insert(*instance_id, instance_payload);
+                    let (task_id, task_size, spilled_table_ids) =
+                        data.task_manager
+                            .spill(&self.context, spilled_table_ids, payload);
+                    for table_id in spilled_table_ids {
+                        spiller
+                            .unsync_data()
+                            .table_data
+                            .get_mut(table_id)
+                            .expect("should exist")
+                            .spill_tasks
+                            .entry(epoch)
+                            .or_default()
+                            .push_front(task_id);
                     }
+                    curr_batch_flush_size += task_size;
                 }
-                curr_batch_flush_size += epoch_data.flush(&self.context, payload);
             }
+            data.check_upload_task_consistency();
             curr_batch_flush_size > 0
         } else {
             false
@@ -1257,16 +1302,11 @@ impl HummockUploader {
     }
 
     pub(crate) fn clear(&mut self) {
-        let max_committed_epoch = self.context.pinned_version.max_committed_epoch();
-        self.max_synced_epoch = max_committed_epoch;
-        self.max_syncing_epoch = max_committed_epoch;
         if let UploaderState::Working(data) = replace(
             &mut self.state,
             UploaderState::Working(UploaderData::default()),
         ) {
-            data.abort(|| {
-                HummockError::other(format!("uploader is reset to {}", max_committed_epoch))
-            });
+            data.abort(|| HummockError::other("uploader is reset"));
         }
 
         self.context.stats.uploader_syncing_epoch_count.set(0);
@@ -1276,100 +1316,122 @@ impl HummockUploader {
         let UploaderState::Working(data) = &mut self.state else {
             return;
         };
-        data.unsync_data.may_destroy_instance(instance_id);
+        if let Some(removed_table_data) = data.unsync_data.may_destroy_instance(instance_id) {
+            data.task_manager.remove_table_spill_tasks(
+                removed_table_data.table_id,
+                removed_table_data
+                    .spill_tasks
+                    .into_values()
+                    .flat_map(|task_ids| task_ids.into_iter())
+                    .filter(|task_id| {
+                        if let Some((_, table_ids)) = data.spilled_data.get_mut(task_id) {
+                            assert!(table_ids.remove(&removed_table_data.table_id));
+                            if table_ids.is_empty() {
+                                data.spilled_data.remove(task_id);
+                            }
+                            false
+                        } else {
+                            true
+                        }
+                    }),
+            )
+        }
+        data.check_upload_task_consistency();
     }
 }
 
 impl UploaderData {
-    /// Poll the syncing task of the syncing data of the oldest epoch. Return `Poll::Ready(None)` if
-    /// there is no syncing data.
-    fn poll_syncing_task(
-        &mut self,
-        cx: &mut Context<'_>,
-        context: &UploaderContext,
-        mut set_max_synced_epoch: impl FnMut(u64),
-    ) -> Poll<Option<Result<Arc<StagingSstableInfo>, ErrState>>> {
-        while let Some(syncing_data) = self.syncing_data.back_mut() {
-            let sstable_info = if let Some(task) = syncing_data.uploading_tasks.back_mut() {
-                let result = ready!(task.poll_result(cx));
-                let _task = syncing_data.uploading_tasks.pop_back().expect("non-empty");
-                let sstable_info = match result {
-                    Ok(sstable_info) => sstable_info,
-                    Err(e) => {
-                        let SyncingData {
-                            sync_epoch,
-                            uploading_tasks,
-                            sync_result_sender,
-                            ..
-                        } = self.syncing_data.pop_back().expect("non-empty");
-                        for task in uploading_tasks {
-                            task.join_handle.abort();
-                        }
-                        send_sync_result(
-                            sync_result_sender,
-                            Err(HummockError::other(format!(
-                                "failed sync task: {:?}",
-                                e.as_report()
-                            ))),
-                        );
-
-                        return Poll::Ready(Some(Err(ErrState {
-                            failed_epoch: sync_epoch,
-                            reason: format!("{:?}", e.as_report()),
-                        })));
+    fn may_notify_sync_task(&mut self, context: &UploaderContext) {
+        while let Some((_, syncing_data)) = self.syncing_data.first_key_value()
+            && syncing_data.remaining_uploading_tasks.is_empty()
+        {
+            let (_, syncing_data) = self.syncing_data.pop_first().expect("non-empty");
+            let SyncingData {
+                sync_epoch,
+                table_ids,
+                remaining_uploading_tasks: _,
+                uploaded,
+                table_watermarks,
+                sync_result_sender,
+            } = syncing_data;
+            context
+                .stats
+                .uploader_syncing_epoch_count
+                .set(self.syncing_data.len() as _);
+
+            for table_id in table_ids {
+                if let Some(table_data) = self.unsync_data.table_data.get_mut(&table_id) {
+                    table_data.ack_synced(sync_epoch);
+                    if table_data.is_empty() {
+                        self.unsync_data.table_data.remove(&table_id);
                     }
-                };
-                syncing_data.uploaded.push_front(sstable_info.clone());
-                self.unsync_data.ack_flushed(&sstable_info);
-                Some(sstable_info)
-            } else {
-                None
-            };
+                }
+            }
 
-            if syncing_data.uploading_tasks.is_empty() {
-                let syncing_data = self.syncing_data.pop_back().expect("non-empty");
-                let SyncingData {
-                    sync_epoch,
-                    uploading_tasks,
-                    uploaded,
+            send_sync_result(
+                sync_result_sender,
+                Ok(SyncedData {
+                    uploaded_ssts: uploaded,
                     table_watermarks,
-                    sync_result_sender,
-                } = syncing_data;
-                assert!(uploading_tasks.is_empty());
-                context
-                    .stats
-                    .uploader_syncing_epoch_count
-                    .set(self.syncing_data.len() as _);
-                set_max_synced_epoch(sync_epoch);
-                send_sync_result(
-                    sync_result_sender,
-                    Ok(SyncedData {
-                        uploaded_ssts: uploaded,
-                        table_watermarks,
-                    }),
-                )
+                }),
+            )
+        }
+    }
+
+    fn check_upload_task_consistency(&self) {
+        #[cfg(debug_assertions)]
+        {
+            let mut spill_task_table_id_from_data: HashMap<_, HashSet<_>> = HashMap::new();
+            for table_data in self.unsync_data.table_data.values() {
+                for task_id in table_data
+                    .spill_tasks
+                    .iter()
+                    .flat_map(|(_, tasks)| tasks.iter())
+                {
+                    assert!(spill_task_table_id_from_data
+                        .entry(*task_id)
+                        .or_default()
+                        .insert(table_data.table_id));
+                }
             }
+            let syncing_task_id_from_data: HashMap<_, HashSet<_>> = self
+                .syncing_data
+                .iter()
+                .filter_map(|(sync_id, data)| {
+                    if data.remaining_uploading_tasks.is_empty() {
+                        None
+                    } else {
+                        Some((*sync_id, data.remaining_uploading_tasks.clone()))
+                    }
+                })
+                .collect();
 
-            if let Some(sstable_info) = sstable_info {
-                return Poll::Ready(Some(Ok(sstable_info)));
+            let mut spill_task_table_id_from_manager: HashMap<_, HashSet<_>> = HashMap::new();
+            for (task_id, (_, table_ids)) in &self.spilled_data {
+                spill_task_table_id_from_manager.insert(*task_id, table_ids.clone());
             }
-        }
-        Poll::Ready(None)
-    }
-
-    /// Poll the success of the oldest spilled task of unsync spill data. Return `Poll::Ready(None)` if
-    /// there is no spilling task.
-    fn poll_spill_task(&mut self, cx: &mut Context<'_>) -> Poll<Option<Arc<StagingSstableInfo>>> {
-        // iterator from older epoch to new epoch so that the spill task are finished in epoch order
-        for epoch_data in self.unsync_data.epoch_data.values_mut() {
-            // if None, there is no spilling task. Search for the unsync data of the next epoch in
-            // the next iteration.
-            if let Some(sstable_info) = ready!(epoch_data.spilled_data.poll_success_spill(cx)) {
-                self.unsync_data.ack_flushed(&sstable_info);
-                return Poll::Ready(Some(sstable_info));
+            let mut syncing_task_from_manager: HashMap<_, HashSet<_>> = HashMap::new();
+            for (task_id, status) in self.task_manager.tasks() {
+                match status {
+                    UploadingTaskStatus::Spilling(table_ids) => {
+                        assert!(spill_task_table_id_from_manager
+                            .insert(task_id, table_ids.clone())
+                            .is_none());
+                    }
+                    UploadingTaskStatus::Sync(sync_id) => {
+                        assert!(syncing_task_from_manager
+                            .entry(*sync_id)
+                            .or_default()
+                            .insert(task_id));
+                    }
+                }
             }
+            assert_eq!(
+                spill_task_table_id_from_data,
+                spill_task_table_id_from_manager
+            );
+            assert_eq!(syncing_task_id_from_data, syncing_task_from_manager);
         }
-        Poll::Ready(None)
     }
 }
 
@@ -1382,254 +1444,82 @@ impl HummockUploader {
                 return Poll::Pending;
             };
 
-            if let Some(result) =
-                ready!(
-                    data.poll_syncing_task(cx, &self.context, |new_synced_epoch| {
-                        Self::set_max_synced_epoch(
-                            &mut self.max_synced_epoch,
-                            self.max_syncing_epoch,
-                            new_synced_epoch,
-                        )
-                    })
-                )
+            if let Some((task_id, status, result)) = ready!(data.task_manager.poll_task_result(cx))
             {
                 match result {
-                    Ok(data) => {
-                        return Poll::Ready(data);
+                    Ok(sst) => {
+                        data.unsync_data.ack_flushed(&sst);
+                        match status {
+                            UploadingTaskStatus::Sync(sync_id) => {
+                                let syncing_data =
+                                    data.syncing_data.get_mut(&sync_id).expect("should exist");
+                                syncing_data.uploaded.push_front(sst.clone());
+                                assert!(syncing_data.remaining_uploading_tasks.remove(&task_id));
+                                data.may_notify_sync_task(&self.context);
+                            }
+                            UploadingTaskStatus::Spilling(table_ids) => {
+                                data.spilled_data.insert(task_id, (sst.clone(), table_ids));
+                            }
+                        }
+                        data.check_upload_task_consistency();
+                        Poll::Ready(sst)
                     }
-                    Err(e) => {
-                        let failed_epoch = e.failed_epoch;
+                    Err((sync_id, e)) => {
+                        let syncing_data =
+                            data.syncing_data.remove(&sync_id).expect("should exist");
+                        let failed_epoch = syncing_data.sync_epoch;
                         let data = must_match!(replace(
                             &mut self.state,
-                            UploaderState::Err(e),
+                            UploaderState::Err(ErrState {
+                                failed_epoch,
+                                reason: e.as_report().to_string(),
+                            }),
                         ), UploaderState::Working(data) => data);
 
+                        let _ = syncing_data
+                            .sync_result_sender
+                            .send(Err(HummockError::other(format!(
+                                "failed to sync: {:?}",
+                                e.as_report()
+                            ))));
+
                         data.abort(|| {
                             HummockError::other(format!(
                                 "previous epoch {} failed to sync",
                                 failed_epoch
                             ))
                         });
-                        return Poll::Pending;
+                        Poll::Pending
                     }
                 }
+            } else {
+                Poll::Pending
             }
-
-            if let Some(sstable_info) = ready!(data.poll_spill_task(cx)) {
-                return Poll::Ready(sstable_info);
-            }
-
-            Poll::Pending
         })
     }
 }
 
 #[cfg(test)]
 pub(crate) mod tests {
-    use std::collections::{HashMap, HashSet, VecDeque};
+    use std::collections::{HashMap, HashSet};
     use std::future::{poll_fn, Future};
     use std::ops::Deref;
+    use std::pin::pin;
     use std::sync::atomic::AtomicUsize;
     use std::sync::atomic::Ordering::SeqCst;
     use std::sync::Arc;
     use std::task::Poll;
 
-    use bytes::Bytes;
-    use futures::future::BoxFuture;
     use futures::FutureExt;
-    use itertools::Itertools;
-    use prometheus::core::GenericGauge;
-    use risingwave_common::catalog::TableId;
-    use risingwave_common::must_match;
-    use risingwave_common::util::epoch::{test_epoch, EpochExt};
-    use risingwave_hummock_sdk::key::{FullKey, TableKey};
-    use risingwave_hummock_sdk::version::HummockVersion;
-    use risingwave_hummock_sdk::{HummockEpoch, LocalSstableInfo};
-    use risingwave_pb::hummock::{KeyRange, SstableInfo};
-    use spin::Mutex;
-    use tokio::spawn;
-    use tokio::sync::mpsc::unbounded_channel;
+    use risingwave_common::util::epoch::EpochExt;
+    use risingwave_hummock_sdk::HummockEpoch;
     use tokio::sync::oneshot;
-    use tokio::task::yield_now;
-
-    use crate::hummock::event_handler::hummock_event_handler::BufferTracker;
-    use crate::hummock::event_handler::uploader::uploader_imm::UploaderImm;
-    use crate::hummock::event_handler::uploader::{
-        get_payload_imm_ids, HummockUploader, SyncedData, UploadTaskInfo, UploadTaskOutput,
-        UploadTaskPayload, UploaderContext, UploaderData, UploaderState, UploadingTask,
-    };
-    use crate::hummock::event_handler::{LocalInstanceId, TEST_LOCAL_INSTANCE_ID};
-    use crate::hummock::local_version::pinned_version::PinnedVersion;
-    use crate::hummock::shared_buffer::shared_buffer_batch::{
-        SharedBufferBatch, SharedBufferBatchId, SharedBufferValue,
-    };
-    use crate::hummock::{HummockError, HummockResult, MemoryLimiter};
-    use crate::mem_table::{ImmId, ImmutableMemtable};
-    use crate::monitor::HummockStateStoreMetrics;
-    use crate::opts::StorageOpts;
-    use crate::store::SealCurrentEpochOptions;
-
-    const INITIAL_EPOCH: HummockEpoch = test_epoch(5);
-    pub(crate) const TEST_TABLE_ID: TableId = TableId { table_id: 233 };
 
-    pub trait UploadOutputFuture =
-        Future<Output = HummockResult<UploadTaskOutput>> + Send + 'static;
-    pub trait UploadFn<Fut: UploadOutputFuture> =
-        Fn(UploadTaskPayload, UploadTaskInfo) -> Fut + Send + Sync + 'static;
-
-    impl HummockUploader {
-        fn data(&self) -> &UploaderData {
-            must_match!(&self.state, UploaderState::Working(data) => data)
-        }
-    }
-
-    fn test_hummock_version(epoch: HummockEpoch) -> HummockVersion {
-        let mut version = HummockVersion::default();
-        version.id = epoch;
-        version.max_committed_epoch = epoch;
-        version
-    }
-
-    fn initial_pinned_version() -> PinnedVersion {
-        PinnedVersion::new(test_hummock_version(INITIAL_EPOCH), unbounded_channel().0)
-    }
-
-    fn dummy_table_key() -> Vec<u8> {
-        vec![b't', b'e', b's', b't']
-    }
-
-    async fn gen_imm_with_limiter(
-        epoch: HummockEpoch,
-        limiter: Option<&MemoryLimiter>,
-    ) -> ImmutableMemtable {
-        let sorted_items = vec![(
-            TableKey(Bytes::from(dummy_table_key())),
-            SharedBufferValue::Delete,
-        )];
-        let size = SharedBufferBatch::measure_batch_size(&sorted_items, None).0;
-        let tracker = match limiter {
-            Some(limiter) => Some(limiter.require_memory(size as u64).await),
-            None => None,
-        };
-        SharedBufferBatch::build_shared_buffer_batch(
-            epoch,
-            0,
-            sorted_items,
-            None,
-            size,
-            TEST_TABLE_ID,
-            tracker,
-        )
-    }
-
-    pub(crate) async fn gen_imm(epoch: HummockEpoch) -> ImmutableMemtable {
-        gen_imm_with_limiter(epoch, None).await
-    }
-
-    fn gen_sstable_info(
-        start_epoch: HummockEpoch,
-        end_epoch: HummockEpoch,
-    ) -> Vec<LocalSstableInfo> {
-        let start_full_key = FullKey::new(TEST_TABLE_ID, TableKey(dummy_table_key()), start_epoch);
-        let end_full_key = FullKey::new(TEST_TABLE_ID, TableKey(dummy_table_key()), end_epoch);
-        let gen_sst_object_id = (start_epoch << 8) + end_epoch;
-        vec![LocalSstableInfo::for_test(SstableInfo {
-            object_id: gen_sst_object_id,
-            sst_id: gen_sst_object_id,
-            key_range: Some(KeyRange {
-                left: start_full_key.encode(),
-                right: end_full_key.encode(),
-                right_exclusive: true,
-            }),
-            table_ids: vec![TEST_TABLE_ID.table_id],
-            ..Default::default()
-        })]
-    }
-
-    fn test_uploader_context<F, Fut>(upload_fn: F) -> UploaderContext
-    where
-        Fut: UploadOutputFuture,
-        F: UploadFn<Fut>,
-    {
-        let config = StorageOpts::default();
-        UploaderContext::new(
-            initial_pinned_version(),
-            Arc::new(move |payload, task_info| spawn(upload_fn(payload, task_info))),
-            BufferTracker::for_test(),
-            &config,
-            Arc::new(HummockStateStoreMetrics::unused()),
-        )
-    }
-
-    fn test_uploader<F, Fut>(upload_fn: F) -> HummockUploader
-    where
-        Fut: UploadOutputFuture,
-        F: UploadFn<Fut>,
-    {
-        let config = StorageOpts {
-            ..Default::default()
-        };
-        HummockUploader::new(
-            Arc::new(HummockStateStoreMetrics::unused()),
-            initial_pinned_version(),
-            Arc::new(move |payload, task_info| spawn(upload_fn(payload, task_info))),
-            BufferTracker::for_test(),
-            &config,
-        )
-    }
-
-    fn dummy_success_upload_output() -> UploadTaskOutput {
-        UploadTaskOutput {
-            new_value_ssts: gen_sstable_info(INITIAL_EPOCH, INITIAL_EPOCH),
-            old_value_ssts: vec![],
-            wait_poll_timer: None,
-        }
-    }
-
-    #[allow(clippy::unused_async)]
-    async fn dummy_success_upload_future(
-        _: UploadTaskPayload,
-        _: UploadTaskInfo,
-    ) -> HummockResult<UploadTaskOutput> {
-        Ok(dummy_success_upload_output())
-    }
-
-    #[allow(clippy::unused_async)]
-    async fn dummy_fail_upload_future(
-        _: UploadTaskPayload,
-        _: UploadTaskInfo,
-    ) -> HummockResult<UploadTaskOutput> {
-        Err(HummockError::other("failed"))
-    }
-
-    impl UploadingTask {
-        fn from_vec(imms: Vec<ImmutableMemtable>, context: &UploaderContext) -> Self {
-            let input = HashMap::from_iter([(
-                TEST_LOCAL_INSTANCE_ID,
-                imms.into_iter().map(UploaderImm::for_test).collect_vec(),
-            )]);
-            Self::new(input, context)
-        }
-    }
-
-    fn get_imm_ids<'a>(
-        imms: impl IntoIterator<Item = &'a ImmutableMemtable>,
-    ) -> HashMap<LocalInstanceId, Vec<SharedBufferBatchId>> {
-        HashMap::from_iter([(
-            TEST_LOCAL_INSTANCE_ID,
-            imms.into_iter().map(|imm| imm.batch_id()).collect_vec(),
-        )])
-    }
-
-    impl HummockUploader {
-        fn local_seal_epoch_for_test(&mut self, instance_id: LocalInstanceId, epoch: HummockEpoch) {
-            self.local_seal_epoch(
-                instance_id,
-                epoch.next_epoch(),
-                SealCurrentEpochOptions::for_test(),
-            );
-        }
-    }
+    use super::test_utils::*;
+    use crate::hummock::event_handler::uploader::{get_payload_imm_ids, SyncedData, UploadingTask};
+    use crate::hummock::event_handler::TEST_LOCAL_INSTANCE_ID;
+    use crate::hummock::HummockError;
+    use crate::opts::StorageOpts;
 
     #[tokio::test]
     pub async fn test_uploading_task_future() {
@@ -1638,11 +1528,11 @@ pub(crate) mod tests {
         let imm = gen_imm(INITIAL_EPOCH).await;
         let imm_size = imm.size();
         let imm_ids = get_imm_ids(vec![&imm]);
-        let task = UploadingTask::from_vec(vec![imm], &uploader_context);
+        let mut task = UploadingTask::from_vec(vec![imm], &uploader_context);
         assert_eq!(imm_size, task.task_info.task_size);
         assert_eq!(imm_ids, task.task_info.imm_ids);
         assert_eq!(vec![INITIAL_EPOCH], task.task_info.epochs);
-        let output = task.await.unwrap();
+        let output = poll_fn(|cx| task.poll_result(cx)).await.unwrap();
         assert_eq!(
             output.sstable_infos(),
             &dummy_success_upload_output().new_value_ssts
@@ -1653,8 +1543,8 @@ pub(crate) mod tests {
 
         let uploader_context = test_uploader_context(dummy_fail_upload_future);
         let imm = gen_imm(INITIAL_EPOCH).await;
-        let task = UploadingTask::from_vec(vec![imm], &uploader_context);
-        let _ = task.await.unwrap_err();
+        let mut task = UploadingTask::from_vec(vec![imm], &uploader_context);
+        let _ = poll_fn(|cx| task.poll_result(cx)).await.unwrap_err();
     }
 
     #[tokio::test]
@@ -1706,6 +1596,7 @@ pub(crate) mod tests {
     async fn test_uploader_basic() {
         let mut uploader = test_uploader(dummy_success_upload_future);
         let epoch1 = INITIAL_EPOCH.next_epoch();
+        uploader.start_epochs_for_test([epoch1]);
         let imm = gen_imm(epoch1).await;
         uploader.init_instance(TEST_LOCAL_INSTANCE_ID, TEST_TABLE_ID, epoch1);
         uploader.add_imm(TEST_LOCAL_INSTANCE_ID, imm.clone());
@@ -1713,12 +1604,12 @@ pub(crate) mod tests {
 
         let (sync_tx, sync_rx) = oneshot::channel();
         uploader.start_sync_epoch(epoch1, sync_tx, HashSet::from_iter([TEST_TABLE_ID]));
-        assert_eq!(epoch1 as HummockEpoch, uploader.max_syncing_epoch);
+        assert_eq!(epoch1 as HummockEpoch, uploader.test_max_syncing_epoch());
         assert_eq!(1, uploader.data().syncing_data.len());
-        let syncing_data = uploader.data().syncing_data.front().unwrap();
+        let (_, syncing_data) = uploader.data().syncing_data.first_key_value().unwrap();
         assert_eq!(epoch1 as HummockEpoch, syncing_data.sync_epoch);
         assert!(syncing_data.uploaded.is_empty());
-        assert!(!syncing_data.uploading_tasks.is_empty());
+        assert!(!syncing_data.remaining_uploading_tasks.is_empty());
 
         let staging_sst = uploader.next_uploaded_sst().await;
         assert_eq!(&vec![epoch1], staging_sst.epochs());
@@ -1752,7 +1643,7 @@ pub(crate) mod tests {
             }
             _ => unreachable!(),
         };
-        assert_eq!(epoch1, uploader.max_synced_epoch());
+        assert_eq!(epoch1, uploader.test_max_synced_epoch());
 
         let new_pinned_version = uploader
             .context
@@ -1768,8 +1659,11 @@ pub(crate) mod tests {
         let epoch1 = INITIAL_EPOCH.next_epoch();
 
         let (sync_tx, sync_rx) = oneshot::channel();
+        uploader.start_epochs_for_test([epoch1]);
+        uploader.init_instance(TEST_LOCAL_INSTANCE_ID, TEST_TABLE_ID, epoch1);
+        uploader.local_seal_epoch_for_test(TEST_LOCAL_INSTANCE_ID, epoch1);
         uploader.start_sync_epoch(epoch1, sync_tx, HashSet::from_iter([TEST_TABLE_ID]));
-        assert_eq!(epoch1, uploader.max_syncing_epoch);
+        assert_eq!(epoch1, uploader.test_max_syncing_epoch());
 
         assert_uploader_pending(&mut uploader).await;
 
@@ -1779,7 +1673,7 @@ pub(crate) mod tests {
             }
             _ => unreachable!(),
         };
-        assert_eq!(epoch1, uploader.max_synced_epoch());
+        assert_eq!(epoch1, uploader.test_max_synced_epoch());
         let new_pinned_version = uploader
             .context
             .pinned_version
@@ -1794,6 +1688,7 @@ pub(crate) mod tests {
         let mut uploader = test_uploader(dummy_success_upload_future);
         let epoch1 = INITIAL_EPOCH.next_epoch();
         let epoch2 = epoch1.next_epoch();
+        uploader.start_epochs_for_test([epoch1, epoch2]);
         let imm = gen_imm(epoch2).await;
         // epoch1 is empty while epoch2 is not. Going to seal empty epoch1.
         uploader.init_instance(TEST_LOCAL_INSTANCE_ID, TEST_TABLE_ID, epoch1);
@@ -1802,7 +1697,7 @@ pub(crate) mod tests {
 
         let (sync_tx, sync_rx) = oneshot::channel();
         uploader.start_sync_epoch(epoch1, sync_tx, HashSet::from_iter([TEST_TABLE_ID]));
-        assert_eq!(epoch1, uploader.max_syncing_epoch);
+        assert_eq!(epoch1, uploader.test_max_syncing_epoch());
 
         assert_uploader_pending(&mut uploader).await;
 
@@ -1812,7 +1707,7 @@ pub(crate) mod tests {
             }
             _ => unreachable!(),
         };
-        assert_eq!(epoch1, uploader.max_synced_epoch());
+        assert_eq!(epoch1, uploader.test_max_synced_epoch());
         let new_pinned_version = uploader
             .context
             .pinned_version
@@ -1825,13 +1720,9 @@ pub(crate) mod tests {
     #[tokio::test]
     async fn test_uploader_poll_empty() {
         let mut uploader = test_uploader(dummy_success_upload_future);
-        let data = must_match!(&mut uploader.state, UploaderState::Working(data) => data);
-        assert!(
-            poll_fn(|cx| data.poll_syncing_task(cx, &uploader.context, |_| unreachable!()))
-                .await
-                .is_none()
-        );
-        assert!(poll_fn(|cx| data.poll_spill_task(cx)).await.is_none());
+        let fut = uploader.next_uploaded_sst();
+        let mut fut = pin!(fut);
+        assert!(poll_fn(|cx| Poll::Ready(fut.as_mut().poll(cx).is_pending())).await);
     }
 
     #[tokio::test]
@@ -1849,28 +1740,31 @@ pub(crate) mod tests {
         let version3 = initial_pinned_version.new_pin_version(test_hummock_version(epoch3));
         let version4 = initial_pinned_version.new_pin_version(test_hummock_version(epoch4));
         let version5 = initial_pinned_version.new_pin_version(test_hummock_version(epoch5));
-        uploader.update_pinned_version(version1);
-        assert_eq!(epoch1, uploader.max_synced_epoch);
-        assert_eq!(epoch1, uploader.max_syncing_epoch);
 
+        uploader.start_epochs_for_test([epoch6]);
         uploader.init_instance(TEST_LOCAL_INSTANCE_ID, TEST_TABLE_ID, epoch6);
+
+        uploader.update_pinned_version(version1);
+        assert_eq!(epoch1, uploader.test_max_synced_epoch());
+        assert_eq!(epoch1, uploader.test_max_syncing_epoch());
+
         let imm = gen_imm(epoch6).await;
         uploader.add_imm(TEST_LOCAL_INSTANCE_ID, imm.clone());
         uploader.update_pinned_version(version2);
-        assert_eq!(epoch2, uploader.max_synced_epoch);
-        assert_eq!(epoch2, uploader.max_syncing_epoch);
+        assert_eq!(epoch2, uploader.test_max_synced_epoch());
+        assert_eq!(epoch2, uploader.test_max_syncing_epoch());
 
         uploader.local_seal_epoch_for_test(TEST_LOCAL_INSTANCE_ID, epoch6);
         uploader.update_pinned_version(version3);
-        assert_eq!(epoch3, uploader.max_synced_epoch);
-        assert_eq!(epoch3, uploader.max_syncing_epoch);
+        assert_eq!(epoch3, uploader.test_max_synced_epoch());
+        assert_eq!(epoch3, uploader.test_max_syncing_epoch());
 
         let (sync_tx, sync_rx) = oneshot::channel();
         uploader.start_sync_epoch(epoch6, sync_tx, HashSet::from_iter([TEST_TABLE_ID]));
-        assert_eq!(epoch6, uploader.max_syncing_epoch);
+        assert_eq!(epoch6, uploader.test_max_syncing_epoch());
         uploader.update_pinned_version(version4);
-        assert_eq!(epoch4, uploader.max_synced_epoch);
-        assert_eq!(epoch6, uploader.max_syncing_epoch);
+        assert_eq!(epoch4, uploader.test_max_synced_epoch());
+        assert_eq!(epoch6, uploader.test_max_syncing_epoch());
 
         let sst = uploader.next_uploaded_sst().await;
         assert_eq!(&get_imm_ids([&imm]), sst.imm_ids());
@@ -1885,84 +1779,8 @@ pub(crate) mod tests {
         }
 
         uploader.update_pinned_version(version5);
-        assert_eq!(epoch6, uploader.max_synced_epoch);
-        assert_eq!(epoch6, uploader.max_syncing_epoch);
-    }
-
-    fn prepare_uploader_order_test(
-        config: &StorageOpts,
-        skip_schedule: bool,
-    ) -> (
-        BufferTracker,
-        HummockUploader,
-        impl Fn(HashMap<LocalInstanceId, Vec<ImmId>>) -> (BoxFuture<'static, ()>, oneshot::Sender<()>),
-    ) {
-        let gauge = GenericGauge::new("test", "test").unwrap();
-        let buffer_tracker = BufferTracker::from_storage_opts(config, gauge);
-        // (the started task send the imm ids of payload, the started task wait for finish notify)
-        #[allow(clippy::type_complexity)]
-        let task_notifier_holder: Arc<
-            Mutex<VecDeque<(oneshot::Sender<UploadTaskInfo>, oneshot::Receiver<()>)>>,
-        > = Arc::new(Mutex::new(VecDeque::new()));
-
-        let new_task_notifier = {
-            let task_notifier_holder = task_notifier_holder.clone();
-            move |imm_ids: HashMap<LocalInstanceId, Vec<ImmId>>| {
-                let (start_tx, start_rx) = oneshot::channel();
-                let (finish_tx, finish_rx) = oneshot::channel();
-                task_notifier_holder
-                    .lock()
-                    .push_front((start_tx, finish_rx));
-                let await_start_future = async move {
-                    let task_info = start_rx.await.unwrap();
-                    assert_eq!(imm_ids, task_info.imm_ids);
-                }
-                .boxed();
-                (await_start_future, finish_tx)
-            }
-        };
-
-        let config = StorageOpts::default();
-        let uploader = HummockUploader::new(
-            Arc::new(HummockStateStoreMetrics::unused()),
-            initial_pinned_version(),
-            Arc::new({
-                move |_, task_info: UploadTaskInfo| {
-                    let task_notifier_holder = task_notifier_holder.clone();
-                    let task_item = task_notifier_holder.lock().pop_back();
-                    let start_epoch = *task_info.epochs.last().unwrap();
-                    let end_epoch = *task_info.epochs.first().unwrap();
-                    assert!(end_epoch >= start_epoch);
-                    spawn(async move {
-                        let ssts = gen_sstable_info(start_epoch, end_epoch);
-                        if !skip_schedule {
-                            let (start_tx, finish_rx) = task_item.unwrap();
-                            start_tx.send(task_info).unwrap();
-                            finish_rx.await.unwrap();
-                        }
-                        Ok(UploadTaskOutput {
-                            new_value_ssts: ssts,
-                            old_value_ssts: vec![],
-                            wait_poll_timer: None,
-                        })
-                    })
-                }
-            }),
-            buffer_tracker.clone(),
-            &config,
-        );
-        (buffer_tracker, uploader, new_task_notifier)
-    }
-
-    async fn assert_uploader_pending(uploader: &mut HummockUploader) {
-        for _ in 0..10 {
-            yield_now().await;
-        }
-        assert!(
-            poll_fn(|cx| Poll::Ready(uploader.next_uploaded_sst().poll_unpin(cx)))
-                .await
-                .is_pending()
-        )
+        assert_eq!(epoch6, uploader.test_max_synced_epoch());
+        assert_eq!(epoch6, uploader.test_max_syncing_epoch());
     }
 
     #[tokio::test]
@@ -1977,6 +1795,9 @@ pub(crate) mod tests {
 
         let epoch1 = INITIAL_EPOCH.next_epoch();
         let epoch2 = epoch1.next_epoch();
+        let epoch3 = epoch2.next_epoch();
+        let epoch4 = epoch3.next_epoch();
+        uploader.start_epochs_for_test([epoch1, epoch2, epoch3, epoch4]);
         let memory_limiter = buffer_tracker.get_memory_limiter().clone();
         let memory_limiter = Some(memory_limiter.deref());
 
@@ -2036,7 +1857,6 @@ pub(crate) mod tests {
         let (sync_tx1, mut sync_rx1) = oneshot::channel();
         uploader.start_sync_epoch(epoch1, sync_tx1, HashSet::from_iter([TEST_TABLE_ID]));
         await_start1_4.await;
-        let epoch3 = epoch2.next_epoch();
 
         uploader.local_seal_epoch_for_test(instance_id1, epoch2);
         uploader.local_seal_epoch_for_test(instance_id2, epoch2);
@@ -2068,7 +1888,6 @@ pub(crate) mod tests {
         // sealed: uploaded sst([imm2])
         // syncing: epoch1: uploading: [imm1_4], [imm1_3], uploaded: sst([imm1_2, imm1_1])
 
-        let epoch4 = epoch3.next_epoch();
         uploader.local_seal_epoch_for_test(instance_id1, epoch3);
         let imm4 = gen_imm_with_limiter(epoch4, memory_limiter).await;
         uploader.add_imm(instance_id1, imm4.clone());
@@ -2134,7 +1953,7 @@ pub(crate) mod tests {
         } else {
             unreachable!("should be sync finish");
         }
-        assert_eq!(epoch2, uploader.max_synced_epoch);
+        assert_eq!(epoch2, uploader.test_max_synced_epoch());
 
         // current uploader state:
         // unsealed: epoch4: imm: imm4
@@ -2188,7 +2007,7 @@ pub(crate) mod tests {
         } else {
             unreachable!("should be sync finish");
         }
-        assert_eq!(epoch4, uploader.max_synced_epoch);
+        assert_eq!(epoch4, uploader.test_max_synced_epoch());
 
         // current uploader state:
         // unsealed: empty
@@ -2213,6 +2032,7 @@ pub(crate) mod tests {
 
         let epoch1 = INITIAL_EPOCH.next_epoch();
         let epoch2 = epoch1.next_epoch();
+        uploader.start_epochs_for_test([epoch1, epoch2]);
         let instance_id1 = 1;
         let instance_id2 = 2;
         let flush_threshold = buffer_tracker.flush_threshold();
diff --git a/src/storage/src/hummock/event_handler/uploader/spiller.rs b/src/storage/src/hummock/event_handler/uploader/spiller.rs
new file mode 100644
index 0000000000000..ba04d85856ace
--- /dev/null
+++ b/src/storage/src/hummock/event_handler/uploader/spiller.rs
@@ -0,0 +1,429 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::{HashMap, HashSet};
+
+use risingwave_common::catalog::TableId;
+use risingwave_hummock_sdk::HummockEpoch;
+
+use crate::hummock::event_handler::uploader::{
+    LocalInstanceUnsyncData, UnsyncData, UnsyncEpochId, UploadTaskInput,
+};
+use crate::hummock::event_handler::LocalInstanceId;
+
+#[derive(Default)]
+struct EpochSpillableDataInfo {
+    instance_ids: HashSet<LocalInstanceId>,
+    payload_size: usize,
+}
+
+pub(super) struct Spiller<'a> {
+    unsync_data: &'a mut UnsyncData,
+    epoch_info: HashMap<UnsyncEpochId, EpochSpillableDataInfo>,
+    unsync_epoch_id_map: HashMap<(HummockEpoch, TableId), UnsyncEpochId>,
+}
+
+impl<'a> Spiller<'a> {
+    pub(super) fn new(unsync_data: &'a mut UnsyncData) -> Self {
+        let unsync_epoch_id_map: HashMap<_, _> = unsync_data
+            .unsync_epochs
+            .iter()
+            .flat_map(|(unsync_epoch_id, table_ids)| {
+                let epoch = unsync_epoch_id.epoch();
+                let unsync_epoch_id = *unsync_epoch_id;
+                table_ids
+                    .iter()
+                    .map(move |table_id| ((epoch, *table_id), unsync_epoch_id))
+            })
+            .collect();
+        let mut epoch_info: HashMap<_, EpochSpillableDataInfo> = HashMap::new();
+        for instance_data in unsync_data
+            .table_data
+            .values()
+            .flat_map(|table_data| table_data.instance_data.values())
+        {
+            if let Some((epoch, spill_size)) = instance_data.spillable_data_info() {
+                let unsync_epoch_id = unsync_epoch_id_map
+                    .get(&(epoch, instance_data.table_id))
+                    .expect("should exist");
+                let epoch_info = epoch_info.entry(*unsync_epoch_id).or_default();
+                assert!(epoch_info.instance_ids.insert(instance_data.instance_id));
+                epoch_info.payload_size += spill_size;
+            }
+        }
+        Self {
+            unsync_data,
+            epoch_info,
+            unsync_epoch_id_map,
+        }
+    }
+
+    pub(super) fn next_spilled_payload(
+        &mut self,
+    ) -> Option<(HummockEpoch, UploadTaskInput, HashSet<TableId>)> {
+        if let Some(unsync_epoch_id) = self
+            .epoch_info
+            .iter()
+            .max_by_key(|(_, info)| info.payload_size)
+            .map(|(unsync_epoch_id, _)| *unsync_epoch_id)
+        {
+            let spill_epoch = unsync_epoch_id.epoch();
+            let spill_info = self
+                .epoch_info
+                .remove(&unsync_epoch_id)
+                .expect("should exist");
+            let epoch_info = &mut self.epoch_info;
+            let mut payload = HashMap::new();
+            let mut spilled_table_ids = HashSet::new();
+            for instance_id in spill_info.instance_ids {
+                let table_id = *self
+                    .unsync_data
+                    .instance_table_id
+                    .get(&instance_id)
+                    .expect("should exist");
+                let instance_data = self
+                    .unsync_data
+                    .table_data
+                    .get_mut(&table_id)
+                    .expect("should exist")
+                    .instance_data
+                    .get_mut(&instance_id)
+                    .expect("should exist");
+                let instance_payload = instance_data.spill(spill_epoch);
+                assert!(!instance_payload.is_empty());
+                payload.insert(instance_id, instance_payload);
+                spilled_table_ids.insert(table_id);
+
+                // update the spill info
+                if let Some((new_spill_epoch, size)) = instance_data.spillable_data_info() {
+                    let new_unsync_epoch_id = self
+                        .unsync_epoch_id_map
+                        .get(&(new_spill_epoch, instance_data.table_id))
+                        .expect("should exist");
+                    let info = epoch_info.entry(*new_unsync_epoch_id).or_default();
+                    assert!(info.instance_ids.insert(instance_id));
+                    info.payload_size += size;
+                }
+            }
+            Some((spill_epoch, payload, spilled_table_ids))
+        } else {
+            None
+        }
+    }
+
+    pub(super) fn unsync_data(&mut self) -> &mut UnsyncData {
+        self.unsync_data
+    }
+}
+
+impl LocalInstanceUnsyncData {
+    fn spillable_data_info(&self) -> Option<(HummockEpoch, usize)> {
+        self.sealed_data
+            .back()
+            .or(self.current_epoch_data.as_ref())
+            .and_then(|epoch_data| {
+                if !epoch_data.is_empty() {
+                    Some((
+                        epoch_data.epoch,
+                        epoch_data.imms.iter().map(|imm| imm.size()).sum(),
+                    ))
+                } else {
+                    None
+                }
+            })
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::collections::{HashMap, HashSet};
+    use std::ops::Deref;
+
+    use futures::future::join_all;
+    use futures::FutureExt;
+    use itertools::Itertools;
+    use risingwave_common::catalog::TableId;
+    use risingwave_common::util::epoch::EpochExt;
+    use tokio::sync::oneshot;
+
+    use crate::hummock::event_handler::uploader::test_utils::*;
+    use crate::opts::StorageOpts;
+    use crate::store::SealCurrentEpochOptions;
+
+    #[tokio::test]
+    async fn test_spill_in_order() {
+        let config = StorageOpts {
+            shared_buffer_capacity_mb: 1024 * 1024,
+            shared_buffer_flush_ratio: 0.0,
+            ..Default::default()
+        };
+        let (buffer_tracker, mut uploader, new_task_notifier) =
+            prepare_uploader_order_test(&config, false);
+
+        let table_id1 = TableId::new(1);
+        let table_id2 = TableId::new(2);
+
+        let instance_id1_1 = 1;
+        let instance_id1_2 = 2;
+        let instance_id2 = 3;
+
+        let epoch1 = INITIAL_EPOCH.next_epoch();
+        let epoch2 = epoch1.next_epoch();
+        let epoch3 = epoch2.next_epoch();
+        let epoch4 = epoch3.next_epoch();
+        let memory_limiter = buffer_tracker.get_memory_limiter().clone();
+        let memory_limiter = Some(memory_limiter.deref());
+
+        // epoch1
+        uploader.start_epoch(epoch1, HashSet::from_iter([table_id1]));
+        uploader.start_epoch(epoch1, HashSet::from_iter([table_id2]));
+
+        uploader.init_instance(instance_id1_1, table_id1, epoch1);
+        uploader.init_instance(instance_id1_2, table_id1, epoch1);
+        uploader.init_instance(instance_id2, table_id2, epoch1);
+
+        // naming: imm<table>_<instance>_<epoch>
+        let imm1_1_1 = gen_imm_inner(table_id1, epoch1, 0, memory_limiter).await;
+        uploader.add_imm(instance_id1_1, imm1_1_1.clone());
+        let imm1_2_1 = gen_imm_inner(table_id1, epoch1, 0, memory_limiter).await;
+        uploader.add_imm(instance_id1_2, imm1_2_1.clone());
+        let imm2_1 = gen_imm_inner(table_id2, epoch1, 0, memory_limiter).await;
+        uploader.add_imm(instance_id2, imm2_1.clone());
+
+        // epoch2
+        uploader.start_epoch(epoch2, HashSet::from_iter([table_id1]));
+        uploader.start_epoch(epoch2, HashSet::from_iter([table_id2]));
+
+        uploader.local_seal_epoch(instance_id1_1, epoch2, SealCurrentEpochOptions::for_test());
+        uploader.local_seal_epoch(instance_id1_2, epoch2, SealCurrentEpochOptions::for_test());
+        uploader.local_seal_epoch(instance_id2, epoch2, SealCurrentEpochOptions::for_test());
+
+        let imms1_1_2 = join_all(
+            [0, 1, 2].map(|offset| gen_imm_inner(table_id1, epoch2, offset, memory_limiter)),
+        )
+        .await;
+        for imm in imms1_1_2.clone() {
+            uploader.add_imm(instance_id1_1, imm);
+        }
+
+        // epoch3
+        uploader.start_epoch(epoch3, HashSet::from_iter([table_id1]));
+        uploader.start_epoch(epoch3, HashSet::from_iter([table_id2]));
+
+        uploader.local_seal_epoch(instance_id1_1, epoch3, SealCurrentEpochOptions::for_test());
+        uploader.local_seal_epoch(instance_id1_2, epoch3, SealCurrentEpochOptions::for_test());
+        uploader.local_seal_epoch(instance_id2, epoch3, SealCurrentEpochOptions::for_test());
+
+        let imms1_2_3 = join_all(
+            [0, 1, 2, 3].map(|offset| gen_imm_inner(table_id1, epoch3, offset, memory_limiter)),
+        )
+        .await;
+        for imm in imms1_2_3.clone() {
+            uploader.add_imm(instance_id1_2, imm);
+        }
+
+        // epoch4
+        uploader.start_epoch(epoch4, HashSet::from_iter([table_id1, table_id2]));
+
+        uploader.local_seal_epoch(instance_id1_1, epoch4, SealCurrentEpochOptions::for_test());
+        uploader.local_seal_epoch(instance_id1_2, epoch4, SealCurrentEpochOptions::for_test());
+        uploader.local_seal_epoch(instance_id2, epoch4, SealCurrentEpochOptions::for_test());
+
+        let imm1_1_4 = gen_imm_inner(table_id1, epoch4, 0, memory_limiter).await;
+        uploader.add_imm(instance_id1_1, imm1_1_4.clone());
+        let imm1_2_4 = gen_imm_inner(table_id1, epoch4, 0, memory_limiter).await;
+        uploader.add_imm(instance_id1_2, imm1_2_4.clone());
+        let imm2_4_1 = gen_imm_inner(table_id2, epoch4, 0, memory_limiter).await;
+        uploader.add_imm(instance_id2, imm2_4_1.clone());
+
+        // uploader state:
+        //          table_id1:                                      table_id2:
+        //          instance_id1_1:         instance_id1_2:         instance_id2
+        //  epoch1  imm1_1_1                imm1_2_1           |    imm2_1      |
+        //  epoch2  imms1_1_2(size 3)                          |                |
+        //  epoch3                          imms_1_2_3(size 4) |                |
+        //  epoch4  imm1_1_4                imm1_2_4                imm2_4_1    |
+
+        let (await_start1_1, finish_tx1_1) = new_task_notifier(HashMap::from_iter([
+            (instance_id1_1, vec![imm1_1_1.batch_id()]),
+            (instance_id1_2, vec![imm1_2_1.batch_id()]),
+        ]));
+        let (await_start3, finish_tx3) = new_task_notifier(HashMap::from_iter([(
+            instance_id1_2,
+            imms1_2_3
+                .iter()
+                .rev()
+                .map(|imm| imm.batch_id())
+                .collect_vec(),
+        )]));
+        let (await_start2, finish_tx2) = new_task_notifier(HashMap::from_iter([(
+            instance_id1_1,
+            imms1_1_2
+                .iter()
+                .rev()
+                .map(|imm| imm.batch_id())
+                .collect_vec(),
+        )]));
+        let (await_start1_4, finish_tx1_4) = new_task_notifier(HashMap::from_iter([
+            (instance_id1_1, vec![imm1_1_4.batch_id()]),
+            (instance_id1_2, vec![imm1_2_4.batch_id()]),
+        ]));
+        let (await_start2_1, finish_tx2_1) = new_task_notifier(HashMap::from_iter([(
+            instance_id2,
+            vec![imm2_1.batch_id()],
+        )]));
+        let (await_start2_4_1, finish_tx2_4_1) = new_task_notifier(HashMap::from_iter([(
+            instance_id2,
+            vec![imm2_4_1.batch_id()],
+        )]));
+
+        uploader.may_flush();
+        await_start1_1.await;
+        await_start3.await;
+        await_start2.await;
+        await_start1_4.await;
+        await_start2_1.await;
+        await_start2_4_1.await;
+
+        assert_uploader_pending(&mut uploader).await;
+
+        let imm2_4_2 = gen_imm_inner(table_id2, epoch4, 1, memory_limiter).await;
+        uploader.add_imm(instance_id2, imm2_4_2.clone());
+
+        uploader.local_seal_epoch(
+            instance_id1_1,
+            u64::MAX,
+            SealCurrentEpochOptions::for_test(),
+        );
+        uploader.local_seal_epoch(
+            instance_id1_2,
+            u64::MAX,
+            SealCurrentEpochOptions::for_test(),
+        );
+        uploader.local_seal_epoch(instance_id2, u64::MAX, SealCurrentEpochOptions::for_test());
+
+        // uploader state:
+        //          table_id1:                                      table_id2:
+        //          instance_id1_1:         instance_id1_2:         instance_id2
+        //  epoch1  spill(imm1_1_1, imm1_2_1, size 2)          |    spill(imm2_1, size 1)               |
+        //  epoch2  spill(imms1_1_2, size 3)                   |                                        |
+        //  epoch3                   spill(imms_1_2_3, size 4) |                                        |
+        //  epoch4  spill(imm1_1_4, imm1_2_4, size 2)               spill(imm2_4_1, size 1), imm2_4_2   |
+
+        let (sync_tx1_1, sync_rx1_1) = oneshot::channel();
+        uploader.start_sync_epoch(epoch1, sync_tx1_1, HashSet::from_iter([table_id1]));
+        let (sync_tx2_1, sync_rx2_1) = oneshot::channel();
+        uploader.start_sync_epoch(epoch2, sync_tx2_1, HashSet::from_iter([table_id1]));
+        let (sync_tx3_1, sync_rx3_1) = oneshot::channel();
+        uploader.start_sync_epoch(epoch3, sync_tx3_1, HashSet::from_iter([table_id1]));
+        let (sync_tx1_2, sync_rx1_2) = oneshot::channel();
+        uploader.start_sync_epoch(epoch1, sync_tx1_2, HashSet::from_iter([table_id2]));
+        let (sync_tx2_2, sync_rx2_2) = oneshot::channel();
+        uploader.start_sync_epoch(epoch2, sync_tx2_2, HashSet::from_iter([table_id2]));
+        let (sync_tx3_2, sync_rx3_2) = oneshot::channel();
+        uploader.start_sync_epoch(epoch3, sync_tx3_2, HashSet::from_iter([table_id2]));
+
+        let (await_start2_4_2, finish_tx2_4_2) = new_task_notifier(HashMap::from_iter([(
+            instance_id2,
+            vec![imm2_4_2.batch_id()],
+        )]));
+
+        finish_tx2_4_1.send(()).unwrap();
+        finish_tx3.send(()).unwrap();
+        finish_tx2.send(()).unwrap();
+        finish_tx1_4.send(()).unwrap();
+        assert_uploader_pending(&mut uploader).await;
+
+        finish_tx1_1.send(()).unwrap();
+        {
+            let imm_ids = HashMap::from_iter([
+                (instance_id1_1, vec![imm1_1_1.batch_id()]),
+                (instance_id1_2, vec![imm1_2_1.batch_id()]),
+            ]);
+            let sst = uploader.next_uploaded_sst().await;
+            assert_eq!(&imm_ids, sst.imm_ids());
+            let synced_data = sync_rx1_1.await.unwrap().unwrap();
+            assert_eq!(synced_data.uploaded_ssts.len(), 1);
+            assert_eq!(&imm_ids, synced_data.uploaded_ssts[0].imm_ids());
+        }
+        {
+            let imm_ids3 = HashMap::from_iter([(
+                instance_id1_2,
+                imms1_2_3
+                    .iter()
+                    .rev()
+                    .map(|imm| imm.batch_id())
+                    .collect_vec(),
+            )]);
+            let imm_ids2 = HashMap::from_iter([(
+                instance_id1_1,
+                imms1_1_2
+                    .iter()
+                    .rev()
+                    .map(|imm| imm.batch_id())
+                    .collect_vec(),
+            )]);
+            let sst = uploader.next_uploaded_sst().await;
+            assert_eq!(&imm_ids3, sst.imm_ids());
+            let sst = uploader.next_uploaded_sst().await;
+            assert_eq!(&imm_ids2, sst.imm_ids());
+            let synced_data = sync_rx2_1.await.unwrap().unwrap();
+            assert_eq!(synced_data.uploaded_ssts.len(), 1);
+            assert_eq!(&imm_ids2, synced_data.uploaded_ssts[0].imm_ids());
+            let synced_data = sync_rx3_1.await.unwrap().unwrap();
+            assert_eq!(synced_data.uploaded_ssts.len(), 1);
+            assert_eq!(&imm_ids3, synced_data.uploaded_ssts[0].imm_ids());
+        }
+        {
+            let imm_ids1_4 = HashMap::from_iter([
+                (instance_id1_1, vec![imm1_1_4.batch_id()]),
+                (instance_id1_2, vec![imm1_2_4.batch_id()]),
+            ]);
+            let imm_ids2_1 = HashMap::from_iter([(instance_id2, vec![imm2_1.batch_id()])]);
+            let imm_ids2_4_1 = HashMap::from_iter([(instance_id2, vec![imm2_4_1.batch_id()])]);
+            finish_tx2_1.send(()).unwrap();
+            let sst = uploader.next_uploaded_sst().await;
+            assert_eq!(&imm_ids1_4, sst.imm_ids());
+
+            // trigger the sync after the spill task is finished and acked to cover the case
+            let (sync_tx4, mut sync_rx4) = oneshot::channel();
+            uploader.start_sync_epoch(epoch4, sync_tx4, HashSet::from_iter([table_id1, table_id2]));
+            await_start2_4_2.await;
+
+            let sst = uploader.next_uploaded_sst().await;
+            assert_eq!(&imm_ids2_1, sst.imm_ids());
+            let sst = uploader.next_uploaded_sst().await;
+            assert_eq!(&imm_ids2_4_1, sst.imm_ids());
+            let synced_data = sync_rx1_2.await.unwrap().unwrap();
+            assert_eq!(synced_data.uploaded_ssts.len(), 1);
+            assert_eq!(&imm_ids2_1, synced_data.uploaded_ssts[0].imm_ids());
+            let synced_data = sync_rx2_2.await.unwrap().unwrap();
+            assert!(synced_data.uploaded_ssts.is_empty());
+            let synced_data = sync_rx3_2.await.unwrap().unwrap();
+            assert!(synced_data.uploaded_ssts.is_empty());
+
+            let imm_ids2_4_2 = HashMap::from_iter([(instance_id2, vec![imm2_4_2.batch_id()])]);
+
+            assert!((&mut sync_rx4).now_or_never().is_none());
+            finish_tx2_4_2.send(()).unwrap();
+            let sst = uploader.next_uploaded_sst().await;
+            assert_eq!(&imm_ids2_4_2, sst.imm_ids());
+            let synced_data = sync_rx4.await.unwrap().unwrap();
+            assert_eq!(synced_data.uploaded_ssts.len(), 3);
+            assert_eq!(&imm_ids2_4_2, synced_data.uploaded_ssts[0].imm_ids());
+            assert_eq!(&imm_ids2_4_1, synced_data.uploaded_ssts[1].imm_ids());
+            assert_eq!(&imm_ids1_4, synced_data.uploaded_ssts[2].imm_ids());
+        }
+    }
+}
diff --git a/src/storage/src/hummock/event_handler/uploader/task_manager.rs b/src/storage/src/hummock/event_handler/uploader/task_manager.rs
new file mode 100644
index 0000000000000..2347be1ed57eb
--- /dev/null
+++ b/src/storage/src/hummock/event_handler/uploader/task_manager.rs
@@ -0,0 +1,187 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::*;
+
+#[derive(Debug)]
+pub(super) enum UploadingTaskStatus {
+    Spilling(HashSet<TableId>),
+    Sync(SyncId),
+}
+
+#[derive(Debug)]
+struct TaskEntry {
+    task: UploadingTask,
+    status: UploadingTaskStatus,
+}
+
+#[derive(Default, Debug)]
+pub(super) struct TaskManager {
+    tasks: HashMap<UploadingTaskId, TaskEntry>,
+    // newer task at the front
+    task_order: VecDeque<UploadingTaskId>,
+    next_task_id: usize,
+}
+
+impl TaskManager {
+    fn add_task(
+        &mut self,
+        task: UploadingTask,
+        status: UploadingTaskStatus,
+    ) -> &UploadingTaskStatus {
+        let task_id = task.task_id;
+        self.task_order.push_front(task.task_id);
+        assert!(self
+            .tasks
+            .insert(task.task_id, TaskEntry { task, status })
+            .is_none());
+        &self.tasks.get(&task_id).expect("should exist").status
+    }
+
+    fn poll_task(
+        &mut self,
+        cx: &mut Context<'_>,
+        task_id: UploadingTaskId,
+    ) -> Poll<Result<Arc<StagingSstableInfo>, (SyncId, HummockError)>> {
+        let entry = self.tasks.get_mut(&task_id).expect("should exist");
+        let result = match &entry.status {
+            UploadingTaskStatus::Spilling(_) => {
+                let sst = ready!(entry.task.poll_ok_with_retry(cx));
+                Ok(sst)
+            }
+            UploadingTaskStatus::Sync(sync_id) => {
+                let result = ready!(entry.task.poll_result(cx));
+                result.map_err(|e| (*sync_id, e))
+            }
+        };
+        Poll::Ready(result)
+    }
+
+    fn get_next_task_id(&mut self) -> UploadingTaskId {
+        let task_id = self.next_task_id;
+        self.next_task_id += 1;
+        UploadingTaskId(task_id)
+    }
+
+    #[expect(clippy::type_complexity)]
+    pub(super) fn poll_task_result(
+        &mut self,
+        cx: &mut Context<'_>,
+    ) -> Poll<
+        Option<(
+            UploadingTaskId,
+            UploadingTaskStatus,
+            Result<Arc<StagingSstableInfo>, (SyncId, HummockError)>,
+        )>,
+    > {
+        if let Some(task_id) = self.task_order.back() {
+            let task_id = *task_id;
+            let result = ready!(self.poll_task(cx, task_id));
+            self.task_order.pop_back();
+            let entry = self.tasks.remove(&task_id).expect("should exist");
+
+            Poll::Ready(Some((task_id, entry.status, result)))
+        } else {
+            Poll::Ready(None)
+        }
+    }
+
+    pub(super) fn abort(self) {
+        for task in self.tasks.into_values() {
+            task.task.join_handle.abort();
+        }
+    }
+
+    pub(super) fn spill(
+        &mut self,
+        context: &UploaderContext,
+        table_ids: HashSet<TableId>,
+        imms: HashMap<LocalInstanceId, Vec<UploaderImm>>,
+    ) -> (UploadingTaskId, usize, &HashSet<TableId>) {
+        assert!(!imms.is_empty());
+        let task = UploadingTask::new(self.get_next_task_id(), imms, context);
+        context.stats.spill_task_counts_from_unsealed.inc();
+        context
+            .stats
+            .spill_task_size_from_unsealed
+            .inc_by(task.task_info.task_size as u64);
+        info!("Spill data. Task: {}", task.get_task_info());
+        let size = task.task_info.task_size;
+        let id = task.task_id;
+        let status = self.add_task(task, UploadingTaskStatus::Spilling(table_ids));
+        (
+            id,
+            size,
+            must_match!(status, UploadingTaskStatus::Spilling(table_ids) => table_ids),
+        )
+    }
+
+    pub(super) fn remove_table_spill_tasks(
+        &mut self,
+        table_id: TableId,
+        task_ids: impl Iterator<Item = UploadingTaskId>,
+    ) {
+        for task_id in task_ids {
+            let entry = self.tasks.get_mut(&task_id).expect("should exist");
+            let empty = must_match!(&mut entry.status, UploadingTaskStatus::Spilling(table_ids) => {
+                assert!(table_ids.remove(&table_id));
+                table_ids.is_empty()
+            });
+            if empty {
+                let task = self.tasks.remove(&task_id).expect("should exist").task;
+                task.join_handle.abort();
+            }
+        }
+    }
+
+    pub(super) fn sync(
+        &mut self,
+        context: &UploaderContext,
+        sync_id: SyncId,
+        unflushed_payload: UploadTaskInput,
+        spill_task_ids: impl Iterator<Item = UploadingTaskId>,
+        sync_table_ids: &HashSet<TableId>,
+    ) -> Option<UploadingTaskId> {
+        let task = if unflushed_payload.is_empty() {
+            None
+        } else {
+            Some(UploadingTask::new(
+                self.get_next_task_id(),
+                unflushed_payload,
+                context,
+            ))
+        };
+
+        for task_id in spill_task_ids {
+            let entry = self.tasks.get_mut(&task_id).expect("should exist");
+            must_match!(&entry.status, UploadingTaskStatus::Spilling(table_ids) => {
+                assert!(table_ids.is_subset(sync_table_ids), "spill table_ids: {table_ids:?}, sync_table_ids: {sync_table_ids:?}");
+            });
+            entry.status = UploadingTaskStatus::Sync(sync_id);
+        }
+
+        task.map(|task| {
+            let id = task.task_id;
+            self.add_task(task, UploadingTaskStatus::Sync(sync_id));
+            id
+        })
+    }
+
+    #[cfg(debug_assertions)]
+    pub(super) fn tasks(&self) -> impl Iterator<Item = (UploadingTaskId, &UploadingTaskStatus)> {
+        self.tasks
+            .iter()
+            .map(|(task_id, entry)| (*task_id, &entry.status))
+    }
+}
diff --git a/src/storage/src/hummock/event_handler/uploader/test_utils.rs b/src/storage/src/hummock/event_handler/uploader/test_utils.rs
new file mode 100644
index 0000000000000..f866ef18f9e22
--- /dev/null
+++ b/src/storage/src/hummock/event_handler/uploader/test_utils.rs
@@ -0,0 +1,348 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![cfg(test)]
+
+use std::collections::{HashMap, HashSet, VecDeque};
+use std::future::{poll_fn, Future};
+use std::sync::atomic::AtomicUsize;
+use std::sync::atomic::Ordering::Relaxed;
+use std::sync::{Arc, LazyLock};
+use std::task::Poll;
+
+use bytes::Bytes;
+use futures::future::BoxFuture;
+use futures::FutureExt;
+use itertools::Itertools;
+use prometheus::core::GenericGauge;
+use risingwave_common::catalog::TableId;
+use risingwave_common::must_match;
+use risingwave_common::util::epoch::{test_epoch, EpochExt};
+use risingwave_hummock_sdk::compaction_group::StaticCompactionGroupId;
+use risingwave_hummock_sdk::key::{FullKey, TableKey};
+use risingwave_hummock_sdk::key_range::KeyRange;
+use risingwave_hummock_sdk::sstable_info::SstableInfo;
+use risingwave_hummock_sdk::version::HummockVersion;
+use risingwave_hummock_sdk::{HummockEpoch, LocalSstableInfo};
+use risingwave_pb::hummock::StateTableInfoDelta;
+use spin::Mutex;
+use tokio::spawn;
+use tokio::sync::mpsc::unbounded_channel;
+use tokio::sync::oneshot;
+use tokio::task::yield_now;
+
+use crate::hummock::event_handler::hummock_event_handler::BufferTracker;
+use crate::hummock::event_handler::uploader::uploader_imm::UploaderImm;
+use crate::hummock::event_handler::uploader::{
+    HummockUploader, TableUnsyncData, UploadTaskInfo, UploadTaskOutput, UploadTaskPayload,
+    UploaderContext, UploaderData, UploaderState, UploadingTask, UploadingTaskId,
+};
+use crate::hummock::event_handler::{LocalInstanceId, TEST_LOCAL_INSTANCE_ID};
+use crate::hummock::local_version::pinned_version::PinnedVersion;
+use crate::hummock::shared_buffer::shared_buffer_batch::{
+    SharedBufferBatch, SharedBufferBatchId, SharedBufferValue,
+};
+use crate::hummock::{HummockError, HummockResult, MemoryLimiter};
+use crate::mem_table::{ImmId, ImmutableMemtable};
+use crate::monitor::HummockStateStoreMetrics;
+use crate::opts::StorageOpts;
+use crate::store::SealCurrentEpochOptions;
+
+pub(crate) const INITIAL_EPOCH: HummockEpoch = test_epoch(5);
+pub(crate) const TEST_TABLE_ID: TableId = TableId { table_id: 233 };
+
+pub trait UploadOutputFuture = Future<Output = HummockResult<UploadTaskOutput>> + Send + 'static;
+pub trait UploadFn<Fut: UploadOutputFuture> =
+    Fn(UploadTaskPayload, UploadTaskInfo) -> Fut + Send + Sync + 'static;
+
+impl HummockUploader {
+    pub(super) fn data(&self) -> &UploaderData {
+        must_match!(&self.state, UploaderState::Working(data) => data)
+    }
+
+    pub(super) fn table_data(&self) -> &TableUnsyncData {
+        self.data()
+            .unsync_data
+            .table_data
+            .get(&TEST_TABLE_ID)
+            .expect("should exist")
+    }
+
+    pub(super) fn test_max_syncing_epoch(&self) -> HummockEpoch {
+        self.table_data().max_sync_epoch().unwrap()
+    }
+
+    pub(super) fn test_max_synced_epoch(&self) -> HummockEpoch {
+        self.table_data().max_synced_epoch.unwrap()
+    }
+}
+
+pub(super) fn test_hummock_version(epoch: HummockEpoch) -> HummockVersion {
+    let mut version = HummockVersion::default();
+    version.id = epoch;
+    version.max_committed_epoch = epoch;
+    version.state_table_info.apply_delta(
+        &HashMap::from_iter([(
+            TEST_TABLE_ID,
+            StateTableInfoDelta {
+                committed_epoch: epoch,
+                safe_epoch: epoch,
+                compaction_group_id: StaticCompactionGroupId::StateDefault as _,
+            },
+        )]),
+        &HashSet::new(),
+    );
+    version
+}
+
+pub(super) fn initial_pinned_version() -> PinnedVersion {
+    PinnedVersion::new(test_hummock_version(INITIAL_EPOCH), unbounded_channel().0)
+}
+
+pub(super) fn dummy_table_key() -> Vec<u8> {
+    vec![b't', b'e', b's', b't']
+}
+
+pub(super) async fn gen_imm_with_limiter(
+    epoch: HummockEpoch,
+    limiter: Option<&MemoryLimiter>,
+) -> ImmutableMemtable {
+    gen_imm_inner(TEST_TABLE_ID, epoch, 0, limiter).await
+}
+
+pub(super) async fn gen_imm_inner(
+    table_id: TableId,
+    epoch: HummockEpoch,
+    spill_offset: u16,
+    limiter: Option<&MemoryLimiter>,
+) -> ImmutableMemtable {
+    let sorted_items = vec![(
+        TableKey(Bytes::from(dummy_table_key())),
+        SharedBufferValue::Delete,
+    )];
+    let size = SharedBufferBatch::measure_batch_size(&sorted_items, None).0;
+    let tracker = match limiter {
+        Some(limiter) => Some(limiter.require_memory(size as u64).await),
+        None => None,
+    };
+    SharedBufferBatch::build_shared_buffer_batch(
+        epoch,
+        spill_offset,
+        sorted_items,
+        None,
+        size,
+        table_id,
+        tracker,
+    )
+}
+
+pub(crate) async fn gen_imm(epoch: HummockEpoch) -> ImmutableMemtable {
+    gen_imm_with_limiter(epoch, None).await
+}
+
+pub(super) fn gen_sstable_info(
+    start_epoch: HummockEpoch,
+    end_epoch: HummockEpoch,
+) -> Vec<LocalSstableInfo> {
+    let start_full_key = FullKey::new(TEST_TABLE_ID, TableKey(dummy_table_key()), start_epoch);
+    let end_full_key = FullKey::new(TEST_TABLE_ID, TableKey(dummy_table_key()), end_epoch);
+    let gen_sst_object_id = (start_epoch << 8) + end_epoch;
+    vec![LocalSstableInfo::for_test(SstableInfo {
+        object_id: gen_sst_object_id,
+        sst_id: gen_sst_object_id,
+        key_range: KeyRange {
+            left: start_full_key.encode().into(),
+            right: end_full_key.encode().into(),
+            right_exclusive: true,
+        },
+        table_ids: vec![TEST_TABLE_ID.table_id],
+        ..Default::default()
+    })]
+}
+
+pub(super) fn test_uploader_context<F, Fut>(upload_fn: F) -> UploaderContext
+where
+    Fut: UploadOutputFuture,
+    F: UploadFn<Fut>,
+{
+    let config = StorageOpts::default();
+    UploaderContext::new(
+        initial_pinned_version(),
+        Arc::new(move |payload, task_info| spawn(upload_fn(payload, task_info))),
+        BufferTracker::for_test(),
+        &config,
+        Arc::new(HummockStateStoreMetrics::unused()),
+    )
+}
+
+pub(super) fn test_uploader<F, Fut>(upload_fn: F) -> HummockUploader
+where
+    Fut: UploadOutputFuture,
+    F: UploadFn<Fut>,
+{
+    let config = StorageOpts {
+        ..Default::default()
+    };
+    HummockUploader::new(
+        Arc::new(HummockStateStoreMetrics::unused()),
+        initial_pinned_version(),
+        Arc::new(move |payload, task_info| spawn(upload_fn(payload, task_info))),
+        BufferTracker::for_test(),
+        &config,
+    )
+}
+
+pub(super) fn dummy_success_upload_output() -> UploadTaskOutput {
+    UploadTaskOutput {
+        new_value_ssts: gen_sstable_info(INITIAL_EPOCH, INITIAL_EPOCH),
+        old_value_ssts: vec![],
+        wait_poll_timer: None,
+    }
+}
+
+#[allow(clippy::unused_async)]
+pub(super) async fn dummy_success_upload_future(
+    _: UploadTaskPayload,
+    _: UploadTaskInfo,
+) -> HummockResult<UploadTaskOutput> {
+    Ok(dummy_success_upload_output())
+}
+
+#[allow(clippy::unused_async)]
+pub(super) async fn dummy_fail_upload_future(
+    _: UploadTaskPayload,
+    _: UploadTaskInfo,
+) -> HummockResult<UploadTaskOutput> {
+    Err(HummockError::other("failed"))
+}
+
+impl UploadingTask {
+    pub(super) fn from_vec(imms: Vec<ImmutableMemtable>, context: &UploaderContext) -> Self {
+        let input = HashMap::from_iter([(
+            TEST_LOCAL_INSTANCE_ID,
+            imms.into_iter().map(UploaderImm::for_test).collect_vec(),
+        )]);
+        static NEXT_TASK_ID: LazyLock<AtomicUsize> = LazyLock::new(|| AtomicUsize::new(0));
+        Self::new(
+            UploadingTaskId(NEXT_TASK_ID.fetch_add(1, Relaxed)),
+            input,
+            context,
+        )
+    }
+}
+
+pub(super) fn get_imm_ids<'a>(
+    imms: impl IntoIterator<Item = &'a ImmutableMemtable>,
+) -> HashMap<LocalInstanceId, Vec<SharedBufferBatchId>> {
+    HashMap::from_iter([(
+        TEST_LOCAL_INSTANCE_ID,
+        imms.into_iter().map(|imm| imm.batch_id()).collect_vec(),
+    )])
+}
+
+impl HummockUploader {
+    pub(super) fn local_seal_epoch_for_test(
+        &mut self,
+        instance_id: LocalInstanceId,
+        epoch: HummockEpoch,
+    ) {
+        self.local_seal_epoch(
+            instance_id,
+            epoch.next_epoch(),
+            SealCurrentEpochOptions::for_test(),
+        );
+    }
+
+    pub(super) fn start_epochs_for_test(&mut self, epochs: impl IntoIterator<Item = HummockEpoch>) {
+        for epoch in epochs {
+            self.start_epoch(epoch, HashSet::from_iter([TEST_TABLE_ID]));
+        }
+    }
+}
+
+pub(crate) fn prepare_uploader_order_test(
+    config: &StorageOpts,
+    skip_schedule: bool,
+) -> (
+    BufferTracker,
+    HummockUploader,
+    impl Fn(HashMap<LocalInstanceId, Vec<ImmId>>) -> (BoxFuture<'static, ()>, oneshot::Sender<()>),
+) {
+    let gauge = GenericGauge::new("test", "test").unwrap();
+    let buffer_tracker = BufferTracker::from_storage_opts(config, gauge);
+    // (the started task send the imm ids of payload, the started task wait for finish notify)
+    #[allow(clippy::type_complexity)]
+    let task_notifier_holder: Arc<
+        Mutex<VecDeque<(oneshot::Sender<UploadTaskInfo>, oneshot::Receiver<()>)>>,
+    > = Arc::new(Mutex::new(VecDeque::new()));
+
+    let new_task_notifier = {
+        let task_notifier_holder = task_notifier_holder.clone();
+        move |imm_ids: HashMap<LocalInstanceId, Vec<ImmId>>| {
+            let (start_tx, start_rx) = oneshot::channel();
+            let (finish_tx, finish_rx) = oneshot::channel();
+            task_notifier_holder
+                .lock()
+                .push_front((start_tx, finish_rx));
+            let await_start_future = async move {
+                let task_info = start_rx.await.unwrap();
+                assert_eq!(imm_ids, task_info.imm_ids);
+            }
+            .boxed();
+            (await_start_future, finish_tx)
+        }
+    };
+
+    let config = StorageOpts::default();
+    let uploader = HummockUploader::new(
+        Arc::new(HummockStateStoreMetrics::unused()),
+        initial_pinned_version(),
+        Arc::new({
+            move |_, task_info: UploadTaskInfo| {
+                let task_notifier_holder = task_notifier_holder.clone();
+                let task_item = task_notifier_holder.lock().pop_back();
+                let start_epoch = *task_info.epochs.last().unwrap();
+                let end_epoch = *task_info.epochs.first().unwrap();
+                assert!(end_epoch >= start_epoch);
+                spawn(async move {
+                    let ssts = gen_sstable_info(start_epoch, end_epoch);
+                    if !skip_schedule {
+                        let (start_tx, finish_rx) = task_item.unwrap();
+                        start_tx.send(task_info).unwrap();
+                        finish_rx.await.unwrap();
+                    }
+                    Ok(UploadTaskOutput {
+                        new_value_ssts: ssts,
+                        old_value_ssts: vec![],
+                        wait_poll_timer: None,
+                    })
+                })
+            }
+        }),
+        buffer_tracker.clone(),
+        &config,
+    );
+    (buffer_tracker, uploader, new_task_notifier)
+}
+
+pub(crate) async fn assert_uploader_pending(uploader: &mut HummockUploader) {
+    for _ in 0..10 {
+        yield_now().await;
+    }
+    assert!(
+        poll_fn(|cx| Poll::Ready(uploader.next_uploaded_sst().poll_unpin(cx)))
+            .await
+            .is_pending()
+    )
+}
diff --git a/src/storage/src/hummock/hummock_meta_client.rs b/src/storage/src/hummock/hummock_meta_client.rs
index 5cf380285cf1e..9663a7787c474 100644
--- a/src/storage/src/hummock/hummock_meta_client.rs
+++ b/src/storage/src/hummock/hummock_meta_client.rs
@@ -18,7 +18,9 @@ use async_trait::async_trait;
 use futures::stream::BoxStream;
 use risingwave_hummock_sdk::version::HummockVersion;
 use risingwave_hummock_sdk::{HummockSstableObjectId, SstObjectIdRange, SyncResult};
-use risingwave_pb::hummock::{HummockSnapshot, SubscribeCompactionEventRequest, VacuumTask};
+use risingwave_pb::hummock::{
+    HummockSnapshot, PbHummockVersion, SubscribeCompactionEventRequest, VacuumTask,
+};
 use risingwave_rpc_client::error::Result;
 use risingwave_rpc_client::{CompactionEventItem, HummockMetaClient, MetaClient};
 use tokio::sync::mpsc::UnboundedSender;
@@ -109,9 +111,13 @@ impl HummockMetaClient for MonitoredHummockMetaClient {
             .await
     }
 
-    async fn trigger_full_gc(&self, sst_retention_time_sec: u64) -> Result<()> {
+    async fn trigger_full_gc(
+        &self,
+        sst_retention_time_sec: u64,
+        prefix: Option<String>,
+    ) -> Result<()> {
         self.meta_client
-            .trigger_full_gc(sst_retention_time_sec)
+            .trigger_full_gc(sst_retention_time_sec, prefix)
             .await
     }
 
@@ -127,4 +133,8 @@ impl HummockMetaClient for MonitoredHummockMetaClient {
     )> {
         self.meta_client.subscribe_compaction_event().await
     }
+
+    async fn get_version_by_epoch(&self, epoch: HummockEpoch) -> Result<PbHummockVersion> {
+        self.meta_client.get_version_by_epoch(epoch).await
+    }
 }
diff --git a/src/storage/src/hummock/iterator/change_log.rs b/src/storage/src/hummock/iterator/change_log.rs
index 7f4d17af6d740..6fc99f29a80f3 100644
--- a/src/storage/src/hummock/iterator/change_log.rs
+++ b/src/storage/src/hummock/iterator/change_log.rs
@@ -18,13 +18,16 @@ use std::ops::Bound::{Excluded, Included, Unbounded};
 use risingwave_common::catalog::TableId;
 use risingwave_common::must_match;
 use risingwave_common::util::epoch::MAX_SPILL_TIMES;
-use risingwave_hummock_sdk::key::{FullKey, SetSlice, TableKeyRange, UserKey, UserKeyRange};
+use risingwave_hummock_sdk::key::{
+    bound_table_key_range, FullKey, SetSlice, TableKeyRange, UserKey, UserKeyRange,
+};
 use risingwave_hummock_sdk::EpochWithGap;
 
 use crate::error::StorageResult;
 use crate::hummock::iterator::{Forward, HummockIterator, MergeIterator};
 use crate::hummock::value::HummockValue;
 use crate::hummock::{HummockResult, SstableIterator};
+use crate::monitor::IterLocalMetricsGuard;
 use crate::store::{ChangeLogValue, StateStoreReadLogItem, StateStoreReadLogItemRef};
 use crate::StateStoreIter;
 
@@ -339,25 +342,37 @@ impl<NI: HummockIterator<Direction = Forward>, OI: HummockIterator<Direction = F
     }
 }
 
+impl Drop for ChangeLogIterator {
+    fn drop(&mut self) {
+        self.inner
+            .new_value_iter
+            .collect_local_statistic(&mut self.stats_guard.local_stats);
+        self.inner
+            .old_value_iter
+            .collect_local_statistic(&mut self.stats_guard.local_stats);
+    }
+}
+
 pub struct ChangeLogIterator {
     inner: ChangeLogIteratorInner<MergeIterator<SstableIterator>, MergeIterator<SstableIterator>>,
     initial_read: bool,
+    stats_guard: IterLocalMetricsGuard,
 }
 
 impl ChangeLogIterator {
     pub async fn new(
         epoch_range: (u64, u64),
-        (start_bound, end_bound): TableKeyRange,
+        table_key_range: TableKeyRange,
         new_value_iter: MergeIterator<SstableIterator>,
         old_value_iter: MergeIterator<SstableIterator>,
         table_id: TableId,
+        stats_guard: IterLocalMetricsGuard,
     ) -> HummockResult<Self> {
-        let make_user_key = |table_key| UserKey {
-            table_id,
-            table_key,
-        };
-        let start_bound = start_bound.map(make_user_key);
-        let end_bound = end_bound.map(make_user_key);
+        let user_key_range_ref = bound_table_key_range(table_id, &table_key_range);
+        let (start_bound, end_bound) = (
+            user_key_range_ref.0.map(|key| key.cloned()),
+            user_key_range_ref.1.map(|key| key.cloned()),
+        );
         let mut inner = ChangeLogIteratorInner::new(
             epoch_range,
             (start_bound, end_bound),
@@ -368,6 +383,7 @@ impl ChangeLogIterator {
         Ok(Self {
             inner,
             initial_read: false,
+            stats_guard,
         })
     }
 }
diff --git a/src/storage/src/hummock/iterator/concat_delete_range_iterator.rs b/src/storage/src/hummock/iterator/concat_delete_range_iterator.rs
index 35a9e53079f9d..a7c5215439bfb 100644
--- a/src/storage/src/hummock/iterator/concat_delete_range_iterator.rs
+++ b/src/storage/src/hummock/iterator/concat_delete_range_iterator.rs
@@ -15,8 +15,8 @@
 use std::future::Future;
 
 use risingwave_hummock_sdk::key::{FullKey, PointRange, UserKey};
+use risingwave_hummock_sdk::sstable_info::SstableInfo;
 use risingwave_hummock_sdk::HummockEpoch;
-use risingwave_pb::hummock::SstableInfo;
 
 use crate::hummock::iterator::DeleteRangeIterator;
 use crate::hummock::sstable_store::SstableStoreRef;
@@ -51,32 +51,26 @@ impl ConcatDeleteRangeIterator {
                     && iter
                         .next_extended_user_key()
                         .left_user_key
-                        .eq(&FullKey::decode(
-                            &self.sstables[self.idx].key_range.as_ref().unwrap().right,
-                        )
-                        .user_key)
+                        .eq(&FullKey::decode(&self.sstables[self.idx].key_range.right).user_key)
                 {
                     // When the last range of the current sstable is equal to the first range of the
                     // next sstable, the `next` method would return two same `PointRange`. So we
                     // must skip one.
                     let exclusive_range_start = iter.next_extended_user_key().is_exclude_left_key;
-                    let last_key_in_sst_start =
-                        iter.next_extended_user_key()
-                            .left_user_key
-                            .eq(&FullKey::decode(
-                                &self.sstables[self.idx + 1].key_range.as_ref().unwrap().left,
-                            )
-                            .user_key);
+                    let last_key_in_sst_start = iter
+                        .next_extended_user_key()
+                        .left_user_key
+                        .eq(&FullKey::decode(&self.sstables[self.idx + 1].key_range.left).user_key);
                     iter.next().await?;
                     if !iter.is_valid() && last_key_in_sst_start {
                         self.seek_idx(self.idx + 1, None).await?;
                         let next_range = self.next_extended_user_key();
                         debug_assert!(self.is_valid());
                         if next_range.is_exclude_left_key == exclusive_range_start
-                            && next_range.left_user_key.eq(&FullKey::decode(
-                                &self.sstables[self.idx].key_range.as_ref().unwrap().left,
-                            )
-                            .user_key)
+                            && next_range
+                                .left_user_key
+                                .eq(&FullKey::decode(&self.sstables[self.idx].key_range.left)
+                                    .user_key)
                         {
                             self.current.as_mut().unwrap().next().await?;
                         }
@@ -158,7 +152,7 @@ impl DeleteRangeIterator for ConcatDeleteRangeIterator {
             let mut idx = self
                 .sstables
                 .partition_point(|sst| {
-                    FullKey::decode(&sst.key_range.as_ref().unwrap().left)
+                    FullKey::decode(&sst.key_range.left)
                         .user_key
                         .le(&target_user_key)
                 })
diff --git a/src/storage/src/hummock/iterator/concat_inner.rs b/src/storage/src/hummock/iterator/concat_inner.rs
index 357d12e32036e..a71a9d9401864 100644
--- a/src/storage/src/hummock/iterator/concat_inner.rs
+++ b/src/storage/src/hummock/iterator/concat_inner.rs
@@ -16,7 +16,7 @@ use std::cmp::Ordering::{Equal, Greater, Less};
 use std::sync::Arc;
 
 use risingwave_hummock_sdk::key::FullKey;
-use risingwave_pb::hummock::SstableInfo;
+use risingwave_hummock_sdk::sstable_info::SstableInfo;
 
 use crate::hummock::iterator::{
     DirectionEnum, HummockIterator, HummockIteratorDirection, ValueMeta,
@@ -27,11 +27,11 @@ use crate::hummock::{HummockResult, SstableIteratorType, SstableStoreRef};
 use crate::monitor::StoreLocalStatistic;
 
 fn smallest_key(sstable_info: &SstableInfo) -> &[u8] {
-    &sstable_info.key_range.as_ref().unwrap().left
+    &sstable_info.key_range.left
 }
 
 fn largest_key(sstable_info: &SstableInfo) -> &[u8] {
-    &sstable_info.key_range.as_ref().unwrap().right
+    &sstable_info.key_range.right
 }
 
 /// Served as the concrete implementation of `ConcatIterator` and `BackwardConcatIterator`.
@@ -160,8 +160,7 @@ impl<TI: SstableIteratorType> HummockIterator for ConcatIteratorInner<TI> {
                 }
                 DirectionEnum::Backward => {
                     let ord = FullKey::decode(largest_key(table)).cmp(&key);
-                    ord == Greater
-                        || (ord == Equal && !table.key_range.as_ref().unwrap().right_exclusive)
+                    ord == Greater || (ord == Equal && !table.key_range.right_exclusive)
                 }
             })
             .saturating_sub(1); // considering the boundary of 0
diff --git a/src/storage/src/hummock/iterator/delete_range_iterator.rs b/src/storage/src/hummock/iterator/delete_range_iterator.rs
index f727990b118fe..bcc2f3e3ea26f 100644
--- a/src/storage/src/hummock/iterator/delete_range_iterator.rs
+++ b/src/storage/src/hummock/iterator/delete_range_iterator.rs
@@ -17,8 +17,8 @@ use std::future::Future;
 
 use risingwave_common::util::epoch::is_max_epoch;
 use risingwave_hummock_sdk::key::{PointRange, UserKey};
+use risingwave_hummock_sdk::sstable_info::SstableInfo;
 use risingwave_hummock_sdk::HummockEpoch;
-use risingwave_pb::hummock::SstableInfo;
 
 use crate::hummock::iterator::concat_delete_range_iterator::ConcatDeleteRangeIterator;
 use crate::hummock::shared_buffer::shared_buffer_batch::SharedBufferDeleteRangeIterator;
diff --git a/src/storage/src/hummock/iterator/mod.rs b/src/storage/src/hummock/iterator/mod.rs
index 8a14efc801062..fdfcd26a3a592 100644
--- a/src/storage/src/hummock/iterator/mod.rs
+++ b/src/storage/src/hummock/iterator/mod.rs
@@ -18,6 +18,7 @@ use std::ops::{Deref, DerefMut};
 use std::sync::Arc;
 
 use more_asserts::{assert_gt, assert_lt};
+use risingwave_hummock_sdk::sstable_info::SstableInfo;
 
 use super::{
     HummockResult, HummockValue, SstableIteratorReadOptions, SstableIteratorType, SstableStoreRef,
@@ -55,7 +56,6 @@ pub use delete_range_iterator::{
     DeleteRangeIterator, ForwardMergeRangeIterator, RangeIteratorTyped,
 };
 use risingwave_common::catalog::TableId;
-use risingwave_pb::hummock::SstableInfo;
 pub use skip_watermark::*;
 
 use crate::hummock::shared_buffer::shared_buffer_batch::SharedBufferBatch;
diff --git a/src/storage/src/hummock/iterator/skip_watermark.rs b/src/storage/src/hummock/iterator/skip_watermark.rs
index 7baba6b2977e7..11ee62dc462f1 100644
--- a/src/storage/src/hummock/iterator/skip_watermark.rs
+++ b/src/storage/src/hummock/iterator/skip_watermark.rs
@@ -16,13 +16,14 @@ use std::cmp::Ordering;
 use std::collections::{BTreeMap, VecDeque};
 
 use bytes::Bytes;
-use risingwave_common::buffer::Bitmap;
 use risingwave_common::catalog::TableId;
-use risingwave_common::hash::{VirtualNode, VnodeBitmapExt};
+use risingwave_common::hash::VirtualNode;
+use risingwave_hummock_sdk::compaction_group::hummock_version_ext::safe_epoch_read_table_watermarks_impl;
 use risingwave_hummock_sdk::key::FullKey;
 use risingwave_hummock_sdk::table_stats::{add_table_stats_map, TableStats, TableStatsMap};
-use risingwave_hummock_sdk::table_watermark::{ReadTableWatermark, WatermarkDirection};
-use risingwave_pb::hummock::PbTableWatermarks;
+use risingwave_hummock_sdk::table_watermark::{
+    ReadTableWatermark, TableWatermarks, WatermarkDirection,
+};
 
 use crate::hummock::iterator::{Forward, HummockIterator, ValueMeta};
 use crate::hummock::value::HummockValue;
@@ -53,7 +54,7 @@ impl<I: HummockIterator<Direction = Forward>> SkipWatermarkIterator<I> {
 
     pub fn from_safe_epoch_watermarks(
         inner: I,
-        safe_epoch_watermarks: &BTreeMap<u32, PbTableWatermarks>,
+        safe_epoch_watermarks: &BTreeMap<u32, TableWatermarks>,
     ) -> Self {
         Self {
             inner,
@@ -185,46 +186,9 @@ impl SkipWatermarkState {
     }
 
     pub fn from_safe_epoch_watermarks(
-        safe_epoch_watermarks: &BTreeMap<u32, PbTableWatermarks>,
+        safe_epoch_watermarks: &BTreeMap<u32, TableWatermarks>,
     ) -> Self {
-        let watermarks = safe_epoch_watermarks
-            .iter()
-            .map(|(table_id, watermarks)| {
-                assert_eq!(watermarks.epoch_watermarks.len(), 1);
-                let vnode_watermarks = &watermarks
-                    .epoch_watermarks
-                    .first()
-                    .expect("should exist")
-                    .watermarks;
-                let mut vnode_watermark_map = BTreeMap::new();
-                for vnode_watermark in vnode_watermarks {
-                    let watermark = Bytes::copy_from_slice(&vnode_watermark.watermark);
-                    for vnode in
-                        Bitmap::from(vnode_watermark.vnode_bitmap.as_ref().expect("should exist"))
-                            .iter_vnodes()
-                    {
-                        assert!(
-                            vnode_watermark_map
-                                .insert(vnode, watermark.clone())
-                                .is_none(),
-                            "duplicate table watermark on vnode {}",
-                            vnode.to_index()
-                        );
-                    }
-                }
-                (
-                    TableId::from(*table_id),
-                    ReadTableWatermark {
-                        direction: if watermarks.is_ascending {
-                            WatermarkDirection::Ascending
-                        } else {
-                            WatermarkDirection::Descending
-                        },
-                        vnode_watermarks: vnode_watermark_map,
-                    },
-                )
-            })
-            .collect();
+        let watermarks = safe_epoch_read_table_watermarks_impl(safe_epoch_watermarks);
         Self::new(watermarks)
     }
 
diff --git a/src/storage/src/hummock/iterator/test_utils.rs b/src/storage/src/hummock/iterator/test_utils.rs
index 9cc6d8dc7fcd2..971e3b7f97c7c 100644
--- a/src/storage/src/hummock/iterator/test_utils.rs
+++ b/src/storage/src/hummock/iterator/test_utils.rs
@@ -22,11 +22,11 @@ use risingwave_common::config::{MetricLevel, ObjectStoreConfig};
 use risingwave_common::hash::VirtualNode;
 use risingwave_common::util::epoch::test_epoch;
 use risingwave_hummock_sdk::key::{prefix_slice_with_vnode, FullKey, TableKey, UserKey};
+use risingwave_hummock_sdk::sstable_info::SstableInfo;
 use risingwave_hummock_sdk::{EpochWithGap, HummockEpoch, HummockSstableObjectId};
 use risingwave_object_store::object::{
     InMemObjectStore, ObjectStore, ObjectStoreImpl, ObjectStoreRef,
 };
-use risingwave_pb::hummock::SstableInfo;
 
 use crate::hummock::shared_buffer::shared_buffer_batch::SharedBufferValue;
 use crate::hummock::sstable::SstableIteratorReadOptions;
diff --git a/src/storage/src/hummock/local_version/pinned_version.rs b/src/storage/src/hummock/local_version/pinned_version.rs
index 5e89a135d825c..a3d7b5544ecfe 100644
--- a/src/storage/src/hummock/local_version/pinned_version.rs
+++ b/src/storage/src/hummock/local_version/pinned_version.rs
@@ -19,10 +19,9 @@ use std::time::{Duration, Instant};
 
 use auto_enums::auto_enum;
 use risingwave_common::catalog::TableId;
+use risingwave_hummock_sdk::level::{Level, Levels};
 use risingwave_hummock_sdk::version::HummockVersion;
 use risingwave_hummock_sdk::{CompactionGroupId, HummockVersionId, INVALID_VERSION_ID};
-use risingwave_pb::hummock::hummock_version::Levels;
-use risingwave_pb::hummock::PbLevel;
 use risingwave_rpc_client::HummockMetaClient;
 use thiserror_ext::AsReport;
 use tokio::sync::mpsc::error::TryRecvError;
@@ -133,7 +132,7 @@ impl PinnedVersion {
         self.version.levels.get(&compaction_group_id).unwrap()
     }
 
-    pub fn levels(&self, table_id: TableId) -> impl Iterator<Item = &PbLevel> {
+    pub fn levels(&self, table_id: TableId) -> impl Iterator<Item = &Level> {
         #[auto_enum(Iterator)]
         match self.version.state_table_info.info().get(&table_id) {
             Some(info) => {
@@ -141,8 +140,6 @@ impl PinnedVersion {
                 let levels = self.levels_by_compaction_groups_id(compaction_group_id);
                 levels
                     .l0
-                    .as_ref()
-                    .unwrap()
                     .sub_levels
                     .iter()
                     .rev()
diff --git a/src/storage/src/hummock/mod.rs b/src/storage/src/hummock/mod.rs
index 21eb4a13e8c31..14ac9532c8cb3 100644
--- a/src/storage/src/hummock/mod.rs
+++ b/src/storage/src/hummock/mod.rs
@@ -19,8 +19,8 @@ use std::sync::Arc;
 
 use bytes::Bytes;
 use risingwave_hummock_sdk::key::{FullKey, TableKey, UserKeyRangeRef};
+use risingwave_hummock_sdk::sstable_info::SstableInfo;
 use risingwave_hummock_sdk::{HummockEpoch, *};
-use risingwave_pb::hummock::SstableInfo;
 
 pub mod block_cache;
 pub use block_cache::*;
@@ -54,6 +54,7 @@ pub mod recent_filter;
 pub use recent_filter::*;
 
 pub mod block_stream;
+mod time_travel_version_cache;
 
 pub use error::*;
 pub use risingwave_common::cache::{CacheableEntry, LookupResult, LruCache};
diff --git a/src/storage/src/hummock/observer_manager.rs b/src/storage/src/hummock/observer_manager.rs
index 0725424aaca76..a9171005aeaa9 100644
--- a/src/storage/src/hummock/observer_manager.rs
+++ b/src/storage/src/hummock/observer_manager.rs
@@ -15,7 +15,7 @@
 use std::collections::HashMap;
 use std::sync::Arc;
 
-use risingwave_common_service::observer_manager::{ObserverState, SubscribeHummock};
+use risingwave_common_service::ObserverState;
 use risingwave_hummock_sdk::version::{HummockVersion, HummockVersionDelta};
 use risingwave_hummock_trace::TraceSpan;
 use risingwave_pb::catalog::Table;
@@ -38,7 +38,9 @@ pub struct HummockObserverNode {
 }
 
 impl ObserverState for HummockObserverNode {
-    type SubscribeType = SubscribeHummock;
+    fn subscribe_type() -> risingwave_pb::meta::SubscribeType {
+        risingwave_pb::meta::SubscribeType::Hummock
+    }
 
     fn handle_notification(&mut self, resp: SubscribeResponse) {
         let Some(info) = resp.info.as_ref() else {
diff --git a/src/storage/src/hummock/sstable/block.rs b/src/storage/src/hummock/sstable/block.rs
index 9ba69882db663..e6436983613dc 100644
--- a/src/storage/src/hummock/sstable/block.rs
+++ b/src/storage/src/hummock/sstable/block.rs
@@ -143,7 +143,7 @@ impl RestartPoint {
 pub struct Block {
     /// Uncompressed entries data, with restart encoded restart points info.
     data: Bytes,
-    /// Uncompressed entried data length.
+    /// Uncompressed entries data length.
     data_len: usize,
 
     /// Table id of this block.
diff --git a/src/storage/src/hummock/sstable/builder.rs b/src/storage/src/hummock/sstable/builder.rs
index 9b007f629f74a..77399eb3e3d08 100644
--- a/src/storage/src/hummock/sstable/builder.rs
+++ b/src/storage/src/hummock/sstable/builder.rs
@@ -19,9 +19,11 @@ use std::sync::Arc;
 use bytes::{Bytes, BytesMut};
 use risingwave_common::util::epoch::is_max_epoch;
 use risingwave_hummock_sdk::key::{user_key, FullKey, MAX_KEY_LEN};
+use risingwave_hummock_sdk::key_range::KeyRange;
+use risingwave_hummock_sdk::sstable_info::SstableInfo;
 use risingwave_hummock_sdk::table_stats::{TableStats, TableStatsMap};
 use risingwave_hummock_sdk::{HummockEpoch, LocalSstableInfo};
-use risingwave_pb::hummock::{BloomFilterType, SstableInfo};
+use risingwave_pb::hummock::BloomFilterType;
 
 use super::utils::CompressionAlgorithm;
 use super::{
@@ -34,6 +36,7 @@ use crate::hummock::value::HummockValue;
 use crate::hummock::{
     Block, BlockHolder, BlockIterator, HummockResult, MemoryLimiter, Xor16FilterBuilder,
 };
+use crate::monitor::CompactorMetrics;
 use crate::opts::StorageOpts;
 
 pub const DEFAULT_SSTABLE_SIZE: usize = 4 * 1024 * 1024;
@@ -85,11 +88,8 @@ impl Default for SstableBuilderOptions {
 
 pub struct SstableBuilderOutput<WO> {
     pub sst_info: LocalSstableInfo,
-    pub bloom_filter_size: usize,
     pub writer_output: WO,
-    pub avg_key_size: usize,
-    pub avg_value_size: usize,
-    pub epoch_count: usize,
+    pub stats: SstableBuilderOutputStats,
 }
 
 pub struct SstableBuilder<W: SstableWriter, F: FilterBuilder> {
@@ -122,6 +122,8 @@ pub struct SstableBuilder<W: SstableWriter, F: FilterBuilder> {
 
     epoch_set: BTreeSet<u64>,
     memory_limiter: Option<Arc<MemoryLimiter>>,
+
+    block_size_vec: Vec<usize>, // for statistics
 }
 
 impl<W: SstableWriter> SstableBuilder<W, Xor16FilterBuilder> {
@@ -167,6 +169,7 @@ impl<W: SstableWriter, F: FilterBuilder> SstableBuilder<W, F> {
             last_table_stats: Default::default(),
             epoch_set: BTreeSet::default(),
             memory_limiter,
+            block_size_vec: Vec::new(),
         }
     }
 
@@ -497,12 +500,12 @@ impl<W: SstableWriter, F: FilterBuilder> SstableBuilder<W, F> {
         let sst_info = SstableInfo {
             object_id: self.sstable_id,
             sst_id: self.sstable_id,
-            bloom_filter_kind: bloom_filter_kind as i32,
-            key_range: Some(risingwave_pb::hummock::KeyRange {
-                left: meta.smallest_key.clone(),
-                right: meta.largest_key.clone(),
+            bloom_filter_kind,
+            key_range: KeyRange {
+                left: Bytes::from(meta.smallest_key.clone()),
+                right: Bytes::from(meta.largest_key.clone()),
                 right_exclusive,
-            }),
+            },
             file_size: meta.estimated_size as u64,
             table_ids: self.table_ids.into_iter().collect(),
             meta_offset: meta.meta_offset,
@@ -524,15 +527,20 @@ impl<W: SstableWriter, F: FilterBuilder> SstableBuilder<W, F> {
             self.epoch_set.len()
         );
         let bloom_filter_size = meta.bloom_filter.len();
+        let sstable_file_size = sst_info.file_size as usize;
 
         let writer_output = self.writer.finish(meta).await?;
         Ok(SstableBuilderOutput::<W::Output> {
-            sst_info: LocalSstableInfo::with_stats(sst_info, self.table_stats),
-            bloom_filter_size,
+            sst_info: LocalSstableInfo::new(sst_info, self.table_stats),
             writer_output,
-            avg_key_size,
-            avg_value_size,
-            epoch_count: self.epoch_set.len(),
+            stats: SstableBuilderOutputStats {
+                bloom_filter_size,
+                avg_key_size,
+                avg_value_size,
+                epoch_count: self.epoch_set.len(),
+                block_size_vec: self.block_size_vec,
+                sstable_file_size,
+            },
         })
     }
 
@@ -560,6 +568,7 @@ impl<W: SstableWriter, F: FilterBuilder> SstableBuilder<W, F> {
             });
         let block = self.block_builder.build();
         self.writer.write_block(block, block_meta).await?;
+        self.block_size_vec.push(block.len());
         self.filter_builder
             .switch_block(self.memory_limiter.clone());
         let data_len = utils::checked_into_u32(self.writer.data_len()).unwrap_or_else(|_| {
@@ -608,6 +617,53 @@ impl<W: SstableWriter, F: FilterBuilder> SstableBuilder<W, F> {
     }
 }
 
+pub struct SstableBuilderOutputStats {
+    bloom_filter_size: usize,
+    avg_key_size: usize,
+    avg_value_size: usize,
+    epoch_count: usize,
+    block_size_vec: Vec<usize>, // for statistics
+    sstable_file_size: usize,
+}
+
+impl SstableBuilderOutputStats {
+    pub fn report_stats(&self, metrics: &Arc<CompactorMetrics>) {
+        if self.bloom_filter_size != 0 {
+            metrics
+                .sstable_bloom_filter_size
+                .observe(self.bloom_filter_size as _);
+        }
+
+        if self.sstable_file_size != 0 {
+            metrics
+                .sstable_file_size
+                .observe(self.sstable_file_size as _);
+        }
+
+        if self.avg_key_size != 0 {
+            metrics.sstable_avg_key_size.observe(self.avg_key_size as _);
+        }
+
+        if self.avg_value_size != 0 {
+            metrics
+                .sstable_avg_value_size
+                .observe(self.avg_value_size as _);
+        }
+
+        if self.epoch_count != 0 {
+            metrics
+                .sstable_distinct_epoch_count
+                .observe(self.epoch_count as _);
+        }
+
+        if !self.block_size_vec.is_empty() {
+            for block_size in &self.block_size_vec {
+                metrics.sstable_block_size.observe(*block_size as _);
+            }
+        }
+    }
+}
+
 #[cfg(test)]
 pub(super) mod tests {
     use std::collections::Bound;
@@ -661,13 +717,10 @@ pub(super) mod tests {
         let output = b.finish().await.unwrap();
         let info = output.sst_info.sst_info;
 
-        assert_bytes_eq!(
-            test_key_of(0).encode(),
-            info.key_range.as_ref().unwrap().left
-        );
+        assert_bytes_eq!(test_key_of(0).encode(), info.key_range.left);
         assert_bytes_eq!(
             test_key_of(TEST_KEYS_COUNT - 1).encode(),
-            info.key_range.as_ref().unwrap().right
+            info.key_range.right
         );
         let (data, meta) = output.writer_output;
         assert_eq!(info.file_size, meta.estimated_size as u64);
diff --git a/src/storage/src/hummock/sstable/multi_builder.rs b/src/storage/src/hummock/sstable/multi_builder.rs
index 4e364ce9f94f5..97b448faec8d7 100644
--- a/src/storage/src/hummock/sstable/multi_builder.rs
+++ b/src/storage/src/hummock/sstable/multi_builder.rs
@@ -278,36 +278,7 @@ where
                 if let Some(progress) = &self.task_progress {
                     progress.inc_ssts_sealed();
                 }
-
-                if builder_output.bloom_filter_size != 0 {
-                    self.compactor_metrics
-                        .sstable_bloom_filter_size
-                        .observe(builder_output.bloom_filter_size as _);
-                }
-
-                if builder_output.sst_info.file_size() != 0 {
-                    self.compactor_metrics
-                        .sstable_file_size
-                        .observe(builder_output.sst_info.file_size() as _);
-                }
-
-                if builder_output.avg_key_size != 0 {
-                    self.compactor_metrics
-                        .sstable_avg_key_size
-                        .observe(builder_output.avg_key_size as _);
-                }
-
-                if builder_output.avg_value_size != 0 {
-                    self.compactor_metrics
-                        .sstable_avg_value_size
-                        .observe(builder_output.avg_value_size as _);
-                }
-
-                if builder_output.epoch_count != 0 {
-                    self.compactor_metrics
-                        .sstable_distinct_epoch_count
-                        .observe(builder_output.epoch_count as _);
-                }
+                builder_output.stats.report_stats(&self.compactor_metrics);
             }
 
             self.concurrent_upload_join_handle
@@ -335,7 +306,10 @@ where
         self.seal_current().await?;
         try_join_all(self.concurrent_upload_join_handle.into_iter())
             .await
-            .map_err(HummockError::sstable_upload_error)?;
+            .map_err(HummockError::sstable_upload_error)?
+            .into_iter()
+            .collect::<HummockResult<Vec<()>>>()?;
+
         Ok(self.sst_outputs)
     }
 }
diff --git a/src/storage/src/hummock/sstable_store.rs b/src/storage/src/hummock/sstable_store.rs
index c654e12bb7d69..cea2c42529ceb 100644
--- a/src/storage/src/hummock/sstable_store.rs
+++ b/src/storage/src/hummock/sstable_store.rs
@@ -26,12 +26,12 @@ use foyer::{
 use futures::{future, StreamExt};
 use itertools::Itertools;
 use risingwave_common::config::StorageMemoryConfig;
+use risingwave_hummock_sdk::sstable_info::SstableInfo;
 use risingwave_hummock_sdk::{HummockSstableObjectId, OBJECT_SUFFIX};
 use risingwave_hummock_trace::TracedCachePolicy;
 use risingwave_object_store::object::{
     ObjectError, ObjectMetadataIter, ObjectStoreRef, ObjectStreamingUploader,
 };
-use risingwave_pb::hummock::SstableInfo;
 use serde::{Deserialize, Serialize};
 use thiserror_ext::AsReport;
 use tokio::task::JoinHandle;
@@ -572,7 +572,7 @@ impl SstableStore {
         sst: &SstableInfo,
         stats: &mut StoreLocalStatistic,
     ) -> impl Future<Output = HummockResult<TableHolder>> + Send + 'static {
-        let object_id = sst.get_object_id();
+        let object_id = sst.object_id;
 
         let entry = self.meta_cache.fetch(object_id, || {
             let store = self.store.clone();
@@ -602,8 +602,10 @@ impl SstableStore {
 
     pub async fn list_object_metadata_from_object_store(
         &self,
+        prefix: Option<String>,
     ) -> HummockResult<ObjectMetadataIter> {
-        let raw_iter = self.store.list(&format!("{}/", self.path)).await?;
+        let list_path = format!("{}/{}", self.path, prefix.unwrap_or("".into()));
+        let raw_iter = self.store.list(&list_path).await?;
         let iter = raw_iter.filter(|r| match r {
             Ok(i) => future::ready(i.key.ends_with(&format!(".{}", OBJECT_SUFFIX))),
             Err(_) => future::ready(true),
@@ -1129,8 +1131,8 @@ mod tests {
     use std::ops::Range;
     use std::sync::Arc;
 
+    use risingwave_hummock_sdk::sstable_info::SstableInfo;
     use risingwave_hummock_sdk::HummockSstableObjectId;
-    use risingwave_pb::hummock::SstableInfo;
 
     use super::{SstableStoreRef, SstableWriterOptions};
     use crate::hummock::iterator::test_utils::{iterator_test_key_of, mock_sstable_store};
diff --git a/src/storage/src/hummock/store/hummock_storage.rs b/src/storage/src/hummock/store/hummock_storage.rs
index 11aa643c3659a..deef394d28826 100644
--- a/src/storage/src/hummock/store/hummock_storage.rs
+++ b/src/storage/src/hummock/store/hummock_storage.rs
@@ -25,17 +25,18 @@ use itertools::Itertools;
 use more_asserts::assert_gt;
 use risingwave_common::catalog::TableId;
 use risingwave_common::util::epoch::is_max_epoch;
-use risingwave_common_service::observer_manager::{NotificationClient, ObserverManager};
+use risingwave_common_service::{NotificationClient, ObserverManager};
 use risingwave_hummock_sdk::key::{
     is_empty_key_range, vnode, vnode_range, TableKey, TableKeyRange,
 };
+use risingwave_hummock_sdk::sstable_info::SstableInfo;
 use risingwave_hummock_sdk::table_watermark::TableWatermarksIndex;
+use risingwave_hummock_sdk::version::HummockVersion;
 use risingwave_hummock_sdk::HummockReadEpoch;
-use risingwave_pb::hummock::SstableInfo;
 use risingwave_rpc_client::HummockMetaClient;
+use thiserror_ext::AsReport;
 use tokio::sync::mpsc::{unbounded_channel, UnboundedSender};
 use tokio::sync::oneshot;
-use tracing::error;
 
 use super::local_hummock_storage::LocalHummockStorage;
 use super::version::{read_filter_for_version, CommittedVersion, HummockVersionReader};
@@ -52,6 +53,7 @@ use crate::hummock::event_handler::{
 use crate::hummock::iterator::change_log::ChangeLogIterator;
 use crate::hummock::local_version::pinned_version::{start_pinned_version_worker, PinnedVersion};
 use crate::hummock::observer_manager::HummockObserverNode;
+use crate::hummock::time_travel_version_cache::SimpleTimeTravelVersionCache;
 use crate::hummock::utils::{validate_safe_epoch, wait_for_epoch};
 use crate::hummock::write_limiter::{WriteLimiter, WriteLimiterRef};
 use crate::hummock::{
@@ -72,7 +74,7 @@ impl Drop for HummockStorageShutdownGuard {
         let _ = self
             .shutdown_sender
             .send(HummockEvent::Shutdown)
-            .inspect_err(|e| error!(event = ?e.0, "unable to send shutdown"));
+            .inspect_err(|e| tracing::debug!(event = ?e.0, "unable to send shutdown"));
     }
 }
 
@@ -115,6 +117,10 @@ pub struct HummockStorage {
     write_limiter: WriteLimiterRef,
 
     compact_await_tree_reg: Option<CompactionAwaitTreeRegRef>,
+
+    hummock_meta_client: Arc<dyn HummockMetaClient>,
+
+    simple_time_travel_version_cache: Arc<SimpleTimeTravelVersionCache>,
 }
 
 pub type ReadVersionTuple = (Vec<ImmutableMemtable>, Vec<SstableInfo>, CommittedVersion);
@@ -232,6 +238,8 @@ impl HummockStorage {
             min_current_epoch,
             write_limiter,
             compact_await_tree_reg: await_tree_reg,
+            hummock_meta_client,
+            simple_time_travel_version_cache: Arc::new(SimpleTimeTravelVersionCache::new()),
         };
 
         tokio::spawn(hummock_event_handler.start_hummock_event_handler_worker());
@@ -254,7 +262,10 @@ impl HummockStorage {
     ) -> StorageResult<Option<Bytes>> {
         let key_range = (Bound::Included(key.clone()), Bound::Included(key.clone()));
 
-        let (key_range, read_version_tuple) = if read_options.read_version_from_backup {
+        let (key_range, read_version_tuple) = if read_options.read_version_from_time_travel {
+            self.build_read_version_by_time_travel(epoch, read_options.table_id, key_range)
+                .await?
+        } else if read_options.read_version_from_backup {
             self.build_read_version_tuple_from_backup(epoch, read_options.table_id, key_range)
                 .await?
         } else {
@@ -276,7 +287,10 @@ impl HummockStorage {
         epoch: u64,
         read_options: ReadOptions,
     ) -> StorageResult<HummockStorageIterator> {
-        let (key_range, read_version_tuple) = if read_options.read_version_from_backup {
+        let (key_range, read_version_tuple) = if read_options.read_version_from_time_travel {
+            self.build_read_version_by_time_travel(epoch, read_options.table_id, key_range)
+                .await?
+        } else if read_options.read_version_from_backup {
             self.build_read_version_tuple_from_backup(epoch, read_options.table_id, key_range)
                 .await?
         } else {
@@ -294,7 +308,10 @@ impl HummockStorage {
         epoch: u64,
         read_options: ReadOptions,
     ) -> StorageResult<HummockStorageRevIterator> {
-        let (key_range, read_version_tuple) = if read_options.read_version_from_backup {
+        let (key_range, read_version_tuple) = if read_options.read_version_from_time_travel {
+            self.build_read_version_by_time_travel(epoch, read_options.table_id, key_range)
+                .await?
+        } else if read_options.read_version_from_backup {
             self.build_read_version_tuple_from_backup(epoch, read_options.table_id, key_range)
                 .await?
         } else {
@@ -306,6 +323,33 @@ impl HummockStorage {
             .await
     }
 
+    async fn build_read_version_by_time_travel(
+        &self,
+        epoch: u64,
+        table_id: TableId,
+        key_range: TableKeyRange,
+    ) -> StorageResult<(TableKeyRange, ReadVersionTuple)> {
+        let fetch = async {
+            let pb_version = self
+                .hummock_meta_client
+                .get_version_by_epoch(epoch)
+                .await
+                .inspect_err(|e| tracing::error!("{}", e.to_report_string()))
+                .map_err(|e| HummockError::meta_error(e.to_report_string()))?;
+            let version = HummockVersion::from_rpc_protobuf(&pb_version);
+            validate_safe_epoch(&version, table_id, epoch)?;
+            let (tx, _rx) = unbounded_channel();
+            Ok(PinnedVersion::new(version, tx))
+        };
+        let version = self
+            .simple_time_travel_version_cache
+            .get_or_insert(epoch, fetch)
+            .await?;
+        Ok(get_committed_read_version_tuple(
+            version, table_id, key_range, epoch,
+        ))
+    }
+
     async fn build_read_version_tuple_from_backup(
         &self,
         epoch: u64,
@@ -454,6 +498,15 @@ impl HummockStorage {
         )
     }
 
+    /// Declare the start of an epoch. This information is provided for spill so that the spill task won't
+    /// include data of two or more syncs.
+    // TODO: remove this method when we support spill task that can include data of more two or more syncs
+    pub fn start_epoch(&self, epoch: HummockEpoch, table_ids: HashSet<TableId>) {
+        let _ = self
+            .hummock_event_sender
+            .send(HummockEvent::StartEpoch { epoch, table_ids });
+    }
+
     pub fn sstable_store(&self) -> SstableStoreRef {
         self.context.sstable_store.clone()
     }
@@ -565,16 +618,14 @@ impl StateStore for HummockStorage {
 
     fn sync(&self, epoch: u64, table_ids: HashSet<TableId>) -> impl SyncFuture {
         let (tx, rx) = oneshot::channel();
-        self.hummock_event_sender
-            .send(HummockEvent::SyncEpoch {
-                new_sync_epoch: epoch,
-                sync_result_sender: tx,
-                table_ids,
-            })
-            .expect("should send success");
+        let _ = self.hummock_event_sender.send(HummockEvent::SyncEpoch {
+            new_sync_epoch: epoch,
+            sync_result_sender: tx,
+            table_ids,
+        });
         rx.map(|recv_result| {
             Ok(recv_result
-                .expect("should wait success")?
+                .map_err(|_| HummockError::other("failed to receive sync result"))??
                 .into_sync_result())
         })
     }
@@ -643,9 +694,6 @@ impl StateStore for HummockStorage {
     }
 }
 
-#[cfg(any(test, feature = "test"))]
-use risingwave_hummock_sdk::version::HummockVersion;
-
 #[cfg(any(test, feature = "test"))]
 impl HummockStorage {
     pub async fn seal_and_sync_epoch(
diff --git a/src/storage/src/hummock/store/local_hummock_storage.rs b/src/storage/src/hummock/store/local_hummock_storage.rs
index a14f3b450adff..ae0d775219c53 100644
--- a/src/storage/src/hummock/store/local_hummock_storage.rs
+++ b/src/storage/src/hummock/store/local_hummock_storage.rs
@@ -19,12 +19,13 @@ use std::sync::Arc;
 
 use await_tree::InstrumentAwait;
 use bytes::Bytes;
-use risingwave_common::buffer::Bitmap;
+use risingwave_common::bitmap::Bitmap;
 use risingwave_common::catalog::{TableId, TableOption};
+use risingwave_common::hash::VirtualNode;
 use risingwave_common::util::epoch::MAX_SPILL_TIMES;
 use risingwave_hummock_sdk::key::{is_empty_key_range, vnode_range, TableKey, TableKeyRange};
+use risingwave_hummock_sdk::sstable_info::SstableInfo;
 use risingwave_hummock_sdk::{EpochWithGap, HummockEpoch};
-use risingwave_pb::hummock::SstableInfo;
 use tracing::{warn, Instrument};
 
 use super::version::{StagingData, VersionUpdate};
@@ -240,27 +241,6 @@ impl LocalHummockStorage {
             )
             .await
     }
-
-    pub async fn may_exist_inner(
-        &self,
-        key_range: TableKeyRange,
-        read_options: ReadOptions,
-    ) -> StorageResult<bool> {
-        if self.mem_table.iter(key_range.clone()).next().is_some() {
-            return Ok(true);
-        }
-
-        let (key_range, read_snapshot) = read_filter_for_version(
-            HummockEpoch::MAX, // Use MAX epoch to make sure we read from latest
-            read_options.table_id,
-            key_range,
-            &self.read_version,
-        )?;
-
-        self.hummock_version_reader
-            .may_exist(key_range, read_options, read_snapshot)
-            .await
-    }
 }
 
 impl StateStoreRead for LocalHummockStorage {
@@ -319,14 +299,6 @@ impl LocalStateStore for LocalHummockStorage {
     type Iter<'a> = LocalHummockStorageIterator<'a>;
     type RevIter<'a> = LocalHummockStorageRevIterator<'a>;
 
-    fn may_exist(
-        &self,
-        key_range: TableKeyRange,
-        read_options: ReadOptions,
-    ) -> impl Future<Output = StorageResult<bool>> + Send + '_ {
-        self.may_exist_inner(key_range, read_options)
-    }
-
     async fn get(
         &self,
         key: TableKey<Bytes>,
@@ -375,6 +347,10 @@ impl LocalStateStore for LocalHummockStorage {
             .await
     }
 
+    fn get_table_watermark(&self, vnode: VirtualNode) -> Option<Bytes> {
+        self.read_version.read().latest_watermark(vnode)
+    }
+
     fn insert(
         &mut self,
         key: TableKey<Bytes>,
diff --git a/src/storage/src/hummock/store/version.rs b/src/storage/src/hummock/store/version.rs
index 9831f57448560..7d0dc49847398 100644
--- a/src/storage/src/hummock/store/version.rs
+++ b/src/storage/src/hummock/store/version.rs
@@ -23,19 +23,21 @@ use bytes::Bytes;
 use futures::future::try_join_all;
 use itertools::Itertools;
 use parking_lot::RwLock;
-use risingwave_common::buffer::Bitmap;
+use risingwave_common::bitmap::Bitmap;
 use risingwave_common::catalog::TableId;
 use risingwave_common::hash::VirtualNode;
 use risingwave_common::util::epoch::MAX_SPILL_TIMES;
+use risingwave_hummock_sdk::change_log::EpochNewChangeLog;
 use risingwave_hummock_sdk::key::{
-    bound_table_key_range, is_empty_key_range, FullKey, TableKey, TableKeyRange, UserKey,
+    bound_table_key_range, FullKey, TableKey, TableKeyRange, UserKey,
 };
 use risingwave_hummock_sdk::key_range::KeyRangeCommon;
+use risingwave_hummock_sdk::sstable_info::SstableInfo;
 use risingwave_hummock_sdk::table_watermark::{
     TableWatermarksIndex, VnodeWatermark, WatermarkDirection,
 };
 use risingwave_hummock_sdk::{EpochWithGap, HummockEpoch, LocalSstableInfo};
-use risingwave_pb::hummock::{EpochNewChangeLog, LevelType, SstableInfo};
+use risingwave_pb::hummock::LevelType;
 use sync_point::sync_point;
 use tracing::warn;
 
@@ -62,7 +64,7 @@ use crate::mem_table::{
     ImmId, ImmutableMemtable, MemTableHummockIterator, MemTableHummockRevIterator,
 };
 use crate::monitor::{
-    GetLocalMetricsGuard, HummockStateStoreMetrics, MayExistLocalMetricsGuard, StoreLocalStatistic,
+    GetLocalMetricsGuard, HummockStateStoreMetrics, IterLocalMetricsGuard, StoreLocalStatistic,
 };
 use crate::store::{gen_min_epoch, ReadLogOptions, ReadOptions};
 
@@ -254,7 +256,13 @@ impl HummockReadVersion {
                 .map(|table_watermarks| {
                     TableWatermarksIndex::new_committed(
                         table_watermarks.clone(),
-                        committed_version.max_committed_epoch(),
+                        committed_version
+                            .version()
+                            .state_table_info
+                            .info()
+                            .get(&table_id)
+                            .expect("should exist")
+                            .committed_epoch,
                     )
                 }),
             staging: StagingVersion {
@@ -315,17 +323,28 @@ impl HummockReadVersion {
 
                     // old data comes first
                     for imm_id in imms.iter().rev() {
-                        let valid = match self.staging.imm.pop_back() {
-                            None => false,
-                            Some(prev_imm_id) => prev_imm_id.batch_id() == *imm_id,
+                        let check_err = match self.staging.imm.pop_back() {
+                            None => Some("empty".to_string()),
+                            Some(prev_imm_id) => {
+                                if prev_imm_id.batch_id() == *imm_id {
+                                    None
+                                } else {
+                                    Some(format!(
+                                        "miss match id {} {}",
+                                        prev_imm_id.batch_id(),
+                                        *imm_id
+                                    ))
+                                }
+                            }
                         };
                         assert!(
-                            valid,
+                            check_err.is_none(),
                             "should be valid staging_sst.size {},
                                     staging_sst.imm_ids {:?},
                                     staging_sst.epochs {:?},
                                     local_imm_ids {:?},
-                                    instance_id {}",
+                                    instance_id {}
+                                    check_err {:?}",
                             staging_sst_ref.imm_size,
                             staging_sst_ref.imm_ids,
                             staging_sst_ref.epochs,
@@ -335,6 +354,7 @@ impl HummockReadVersion {
                                 .map(|imm| imm.batch_id())
                                 .collect_vec(),
                             self.instance_id,
+                            check_err
                         );
                     }
 
@@ -343,54 +363,78 @@ impl HummockReadVersion {
             },
 
             VersionUpdate::CommittedSnapshot(committed_version) => {
-                let max_committed_epoch = committed_version.max_committed_epoch();
-                self.committed = committed_version;
-
+                if let Some(info) = committed_version
+                    .version()
+                    .state_table_info
+                    .info()
+                    .get(&self.table_id)
                 {
-                    // TODO: remove it when support update staging local_sst
-                    self.staging
-                        .imm
-                        .retain(|imm| imm.min_epoch() > max_committed_epoch);
+                    let committed_epoch = info.committed_epoch;
+                    self.staging.imm.retain(|imm| {
+                        if self.is_replicated {
+                            imm.min_epoch() > committed_epoch
+                        } else {
+                            assert!(imm.min_epoch() > committed_epoch);
+                            true
+                        }
+                    });
 
                     self.staging.sst.retain(|sst| {
-                        sst.epochs.first().expect("epochs not empty") > &max_committed_epoch
+                        sst.epochs.first().expect("epochs not empty") > &committed_epoch
                     });
 
                     // check epochs.last() > MCE
                     assert!(self.staging.sst.iter().all(|sst| {
-                        sst.epochs.last().expect("epochs not empty") > &max_committed_epoch
+                        sst.epochs.last().expect("epochs not empty") > &committed_epoch
                     }));
-                }
 
-                if let Some(committed_watermarks) = self
-                    .committed
-                    .version()
-                    .table_watermarks
-                    .get(&self.table_id)
-                {
-                    if let Some(watermark_index) = &mut self.table_watermarks {
-                        watermark_index.apply_committed_watermarks(
-                            committed_watermarks.clone(),
-                            self.committed.max_committed_epoch(),
-                        );
-                    } else {
-                        self.table_watermarks = Some(TableWatermarksIndex::new_committed(
-                            committed_watermarks.clone(),
-                            self.committed.max_committed_epoch(),
-                        ));
+                    if let Some(committed_watermarks) = self
+                        .committed
+                        .version()
+                        .table_watermarks
+                        .get(&self.table_id)
+                    {
+                        if let Some(watermark_index) = &mut self.table_watermarks {
+                            watermark_index.apply_committed_watermarks(
+                                committed_watermarks.clone(),
+                                committed_epoch,
+                            );
+                        } else {
+                            self.table_watermarks = Some(TableWatermarksIndex::new_committed(
+                                committed_watermarks.clone(),
+                                committed_epoch,
+                            ));
+                        }
                     }
                 }
+
+                self.committed = committed_version;
             }
             VersionUpdate::NewTableWatermark {
                 direction,
                 epoch,
                 vnode_watermarks,
-            } => self
-                .table_watermarks
-                .get_or_insert_with(|| {
-                    TableWatermarksIndex::new(direction, self.committed.max_committed_epoch())
-                })
-                .add_epoch_watermark(epoch, Arc::from(vnode_watermarks), direction),
+            } => {
+                if let Some(watermark_index) = &mut self.table_watermarks {
+                    watermark_index.add_epoch_watermark(
+                        epoch,
+                        Arc::from(vnode_watermarks),
+                        direction,
+                    );
+                } else {
+                    self.table_watermarks = Some(TableWatermarksIndex::new(
+                        direction,
+                        epoch,
+                        vnode_watermarks,
+                        self.committed
+                            .version()
+                            .state_table_info
+                            .info()
+                            .get(&self.table_id)
+                            .map(|info| info.committed_epoch),
+                    ));
+                }
+            }
         }
     }
 
@@ -412,6 +456,12 @@ impl HummockReadVersion {
         }
     }
 
+    pub fn latest_watermark(&self, vnode: VirtualNode) -> Option<Bytes> {
+        self.table_watermarks
+            .as_ref()
+            .and_then(|watermark_index| watermark_index.latest_watermark(vnode))
+    }
+
     pub fn is_replicated(&self) -> bool {
         self.is_replicated
     }
@@ -566,7 +616,7 @@ impl HummockVersionReader {
                 continue;
             }
 
-            match level.level_type() {
+            match level.level_type {
                 LevelType::Overlapping | LevelType::Unspecified => {
                     let sstable_infos = prune_overlapping_ssts(
                         &level.table_infos,
@@ -602,8 +652,6 @@ impl HummockVersionReader {
                     table_info_idx = table_info_idx.saturating_sub(1);
                     let ord = level.table_infos[table_info_idx]
                         .key_range
-                        .as_ref()
-                        .unwrap()
                         .compare_right_with_user_key(full_key.user_key.as_ref());
                     // the case that the key falls into the gap between two ssts
                     if ord == Ordering::Less {
@@ -816,7 +864,7 @@ impl HummockVersionReader {
                 continue;
             }
 
-            if level.level_type == LevelType::Nonoverlapping as i32 {
+            if level.level_type == LevelType::Nonoverlapping {
                 let table_infos = prune_nonoverlapping_ssts(&level.table_infos, user_key_range_ref);
                 let sstables = table_infos
                     .filter(|sstable_info| {
@@ -881,7 +929,7 @@ impl HummockVersionReader {
                         .sstable_store
                         .sstable(sstable_info, local_stats)
                         .await?;
-                    assert_eq!(sstable_info.get_object_id(), sstable.id);
+                    assert_eq!(sstable_info.object_id, sstable.id);
                     if let Some(dist_hash) = bloom_filter_prefix_hash.as_ref() {
                         if !hit_sstable_bloom_filter(
                             &sstable,
@@ -913,135 +961,6 @@ impl HummockVersionReader {
         Ok(())
     }
 
-    // Note: this method will not check the kv tomestones and delete range tomestones
-    pub async fn may_exist(
-        &self,
-        table_key_range: TableKeyRange,
-        read_options: ReadOptions,
-        read_version_tuple: ReadVersionTuple,
-    ) -> StorageResult<bool> {
-        if is_empty_key_range(&table_key_range) {
-            return Ok(false);
-        }
-
-        let table_id = read_options.table_id;
-        let (imms, uncommitted_ssts, committed_version) = read_version_tuple;
-        let mut stats_guard =
-            MayExistLocalMetricsGuard::new(self.state_store_metrics.clone(), table_id);
-
-        // 1. check staging data
-        for imm in &imms {
-            if imm.range_exists(&table_key_range) {
-                return Ok(true);
-            }
-        }
-
-        let user_key_range = bound_table_key_range(read_options.table_id, &table_key_range);
-        let user_key_range_ref = (
-            user_key_range.0.as_ref().map(UserKey::as_ref),
-            user_key_range.1.as_ref().map(UserKey::as_ref),
-        );
-        let bloom_filter_prefix_hash = if let Some(prefix_hint) = read_options.prefix_hint {
-            Sstable::hash_for_bloom_filter(&prefix_hint, table_id.table_id)
-        } else {
-            // only use `table_key_range` to see whether all SSTs are filtered out
-            // without looking at bloom filter because prefix_hint is not provided
-            if !uncommitted_ssts.is_empty() {
-                // uncommitted_ssts is already pruned by `table_key_range` so no extra check is
-                // needed.
-                return Ok(true);
-            }
-            for level in committed_version.levels(table_id) {
-                match level.level_type() {
-                    LevelType::Overlapping | LevelType::Unspecified => {
-                        if prune_overlapping_ssts(&level.table_infos, table_id, &table_key_range)
-                            .next()
-                            .is_some()
-                        {
-                            return Ok(true);
-                        }
-                    }
-                    LevelType::Nonoverlapping => {
-                        if prune_nonoverlapping_ssts(&level.table_infos, user_key_range_ref)
-                            .next()
-                            .is_some()
-                        {
-                            return Ok(true);
-                        }
-                    }
-                }
-            }
-            return Ok(false);
-        };
-
-        // 2. order guarantee: imm -> sst
-        for local_sst in &uncommitted_ssts {
-            stats_guard.local_stats.may_exist_check_sstable_count += 1;
-            if hit_sstable_bloom_filter(
-                self.sstable_store
-                    .sstable(local_sst, &mut stats_guard.local_stats)
-                    .await?
-                    .as_ref(),
-                &user_key_range_ref,
-                bloom_filter_prefix_hash,
-                &mut stats_guard.local_stats,
-            ) {
-                return Ok(true);
-            }
-        }
-
-        // 3. read from committed_version sst file
-        // Because SST meta records encoded key range,
-        // the filter key needs to be encoded as well.
-        assert!(committed_version.is_valid());
-        for level in committed_version.levels(table_id) {
-            if level.table_infos.is_empty() {
-                continue;
-            }
-            match level.level_type() {
-                LevelType::Overlapping | LevelType::Unspecified => {
-                    let sstable_infos =
-                        prune_overlapping_ssts(&level.table_infos, table_id, &table_key_range);
-                    for sstable_info in sstable_infos {
-                        stats_guard.local_stats.may_exist_check_sstable_count += 1;
-                        if hit_sstable_bloom_filter(
-                            self.sstable_store
-                                .sstable(sstable_info, &mut stats_guard.local_stats)
-                                .await?
-                                .as_ref(),
-                            &user_key_range_ref,
-                            bloom_filter_prefix_hash,
-                            &mut stats_guard.local_stats,
-                        ) {
-                            return Ok(true);
-                        }
-                    }
-                }
-                LevelType::Nonoverlapping => {
-                    let table_infos =
-                        prune_nonoverlapping_ssts(&level.table_infos, user_key_range_ref);
-
-                    for table_info in table_infos {
-                        stats_guard.local_stats.may_exist_check_sstable_count += 1;
-                        if hit_sstable_bloom_filter(
-                            self.sstable_store
-                                .sstable(table_info, &mut stats_guard.local_stats)
-                                .await?
-                                .as_ref(),
-                            &user_key_range_ref,
-                            bloom_filter_prefix_hash,
-                            &mut stats_guard.local_stats,
-                        ) {
-                            return Ok(true);
-                        }
-                    }
-                }
-            }
-        }
-
-        Ok(false)
-    }
-
     pub async fn iter_log(
         &self,
         version: PinnedVersion,
@@ -1056,6 +975,17 @@ impl HummockVersionReader {
                 static EMPTY_VEC: Vec<EpochNewChangeLog> = Vec::new();
                 &EMPTY_VEC[..]
             };
+        if let Some(max_epoch_change_log) = change_log.last() {
+            let (_, max_epoch) = epoch_range;
+            if !max_epoch_change_log.epochs.contains(&max_epoch) {
+                warn!(
+                    max_epoch,
+                    change_log_epochs = ?change_log.iter().flat_map(|epoch_log| epoch_log.epochs.iter()).collect_vec(),
+                    table_id = options.table_id.table_id,
+                    "max_epoch does not exist"
+                );
+            }
+        }
         let read_options = Arc::new(SstableIteratorReadOptions {
             cache_policy: Default::default(),
             must_iterated_end_user_key: None,
@@ -1067,6 +997,7 @@ impl HummockVersionReader {
             ssts: impl Iterator<Item = &SstableInfo>,
             sstable_store: &SstableStoreRef,
             read_options: Arc<SstableIteratorReadOptions>,
+            local_stat: &mut StoreLocalStatistic,
         ) -> HummockResult<MergeIterator<SstableIterator>> {
             let iters = try_join_all(ssts.map(|sst| {
                 let sstable_store = sstable_store.clone();
@@ -1074,16 +1005,23 @@ impl HummockVersionReader {
                 async move {
                     let mut local_stat = StoreLocalStatistic::default();
                     let table_holder = sstable_store.sstable(sst, &mut local_stat).await?;
-                    Ok::<_, HummockError>(SstableIterator::new(
-                        table_holder,
-                        sstable_store,
-                        read_options,
+                    Ok::<_, HummockError>((
+                        SstableIterator::new(table_holder, sstable_store, read_options),
+                        local_stat,
                     ))
                 }
             }))
             .await?;
-            Ok::<_, HummockError>(MergeIterator::new(iters))
+            Ok::<_, HummockError>(MergeIterator::new(iters.into_iter().map(
+                |(iter, stats)| {
+                    local_stat.add(&stats);
+                    iter
+                },
+            )))
         }
+
+        let mut local_stat = StoreLocalStatistic::default();
+
         let new_value_iter = make_iter(
             change_log
                 .iter()
@@ -1091,6 +1029,7 @@ impl HummockVersionReader {
                 .filter(|sst| filter_single_sst(sst, options.table_id, &key_range)),
             &self.sstable_store,
             read_options.clone(),
+            &mut local_stat,
         )
         .await?;
         let old_value_iter = make_iter(
@@ -1100,6 +1039,7 @@ impl HummockVersionReader {
                 .filter(|sst| filter_single_sst(sst, options.table_id, &key_range)),
             &self.sstable_store,
             read_options.clone(),
+            &mut local_stat,
         )
         .await?;
         ChangeLogIterator::new(
@@ -1108,6 +1048,11 @@ impl HummockVersionReader {
             new_value_iter,
             old_value_iter,
             options.table_id,
+            IterLocalMetricsGuard::new(
+                self.state_store_metrics.clone(),
+                options.table_id,
+                local_stat,
+            ),
         )
         .await
     }
diff --git a/src/storage/src/hummock/test_utils.rs b/src/storage/src/hummock/test_utils.rs
index cfed5f0eb7824..53b58d895c808 100644
--- a/src/storage/src/hummock/test_utils.rs
+++ b/src/storage/src/hummock/test_utils.rs
@@ -27,8 +27,9 @@ use risingwave_common::config::EvictionConfig;
 use risingwave_common::hash::VirtualNode;
 use risingwave_common::util::epoch::test_epoch;
 use risingwave_hummock_sdk::key::{FullKey, PointRange, TableKey, UserKey};
+use risingwave_hummock_sdk::key_range::KeyRange;
+use risingwave_hummock_sdk::sstable_info::SstableInfo;
 use risingwave_hummock_sdk::{EpochWithGap, HummockEpoch, HummockSstableObjectId};
-use risingwave_pb::hummock::{KeyRange, SstableInfo};
 
 use super::iterator::test_utils::iterator_test_table_key_of;
 use super::{
@@ -112,11 +113,11 @@ pub fn gen_dummy_sst_info(
     SstableInfo {
         object_id: id,
         sst_id: id,
-        key_range: Some(KeyRange {
-            left: FullKey::for_test(table_id, min_table_key, epoch).encode(),
-            right: FullKey::for_test(table_id, max_table_key, epoch).encode(),
+        key_range: KeyRange {
+            left: Bytes::from(FullKey::for_test(table_id, min_table_key, epoch).encode()),
+            right: Bytes::from(FullKey::for_test(table_id, max_table_key, epoch).encode()),
             right_exclusive: false,
-        }),
+        },
         file_size,
         table_ids: vec![table_id.table_id],
         uncompressed_file_size: file_size,
@@ -189,11 +190,11 @@ pub async fn put_sst(
     let sst = SstableInfo {
         object_id: sst_object_id,
         sst_id: sst_object_id,
-        key_range: Some(KeyRange {
-            left: meta.smallest_key.clone(),
-            right: meta.largest_key.clone(),
+        key_range: KeyRange {
+            left: Bytes::from(meta.smallest_key.clone()),
+            right: Bytes::from(meta.largest_key.clone()),
             right_exclusive: false,
-        }),
+        },
         file_size: meta.estimated_size as u64,
         meta_offset: meta.meta_offset,
         uncompressed_file_size: meta.estimated_size as u64,
diff --git a/src/storage/src/hummock/time_travel_version_cache.rs b/src/storage/src/hummock/time_travel_version_cache.rs
new file mode 100644
index 0000000000000..08ad70ab44fa4
--- /dev/null
+++ b/src/storage/src/hummock/time_travel_version_cache.rs
@@ -0,0 +1,72 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::future::Future;
+
+use moka::sync::Cache;
+use risingwave_hummock_sdk::HummockEpoch;
+use tokio::sync::Mutex;
+
+use crate::hummock::local_version::pinned_version::PinnedVersion;
+use crate::hummock::HummockResult;
+
+/// A naive cache to reduce number of RPC sent to meta node.
+pub struct SimpleTimeTravelVersionCache {
+    inner: Mutex<SimpleTimeTravelVersionCacheInner>,
+}
+
+impl SimpleTimeTravelVersionCache {
+    pub fn new() -> Self {
+        Self {
+            inner: Mutex::new(SimpleTimeTravelVersionCacheInner::new()),
+        }
+    }
+
+    pub async fn get_or_insert(
+        &self,
+        epoch: HummockEpoch,
+        fetch: impl Future<Output = HummockResult<PinnedVersion>>,
+    ) -> HummockResult<PinnedVersion> {
+        let mut guard = self.inner.lock().await;
+        if let Some(v) = guard.get(&epoch) {
+            return Ok(v);
+        }
+        let version = fetch.await?;
+        guard.add(epoch, version);
+        Ok(guard.get(&epoch).unwrap())
+    }
+}
+
+struct SimpleTimeTravelVersionCacheInner {
+    cache: Cache<HummockEpoch, PinnedVersion>,
+}
+
+impl SimpleTimeTravelVersionCacheInner {
+    fn new() -> Self {
+        let capacity = std::env::var("RW_HUMMOCK_TIME_TRAVEL_CACHE_SIZE")
+            .unwrap_or_else(|_| "10".into())
+            .parse()
+            .unwrap();
+        let cache = Cache::builder().max_capacity(capacity).build();
+        Self { cache }
+    }
+
+    fn get(&self, epoch: &HummockEpoch) -> Option<PinnedVersion> {
+        self.cache.get(epoch)
+    }
+
+    fn add(&mut self, epoch: HummockEpoch, version: PinnedVersion) {
+        self.cache.insert(epoch, version);
+    }
+}
diff --git a/src/storage/src/hummock/utils.rs b/src/storage/src/hummock/utils.rs
index 4c270ee736b97..1e1b230964020 100644
--- a/src/storage/src/hummock/utils.rs
+++ b/src/storage/src/hummock/utils.rs
@@ -28,9 +28,9 @@ use risingwave_common::catalog::{TableId, TableOption};
 use risingwave_hummock_sdk::key::{
     bound_table_key_range, EmptySliceRef, FullKey, TableKey, UserKey,
 };
+use risingwave_hummock_sdk::sstable_info::SstableInfo;
 use risingwave_hummock_sdk::version::HummockVersion;
 use risingwave_hummock_sdk::{can_concat, HummockEpoch};
-use risingwave_pb::hummock::SstableInfo;
 use tokio::sync::oneshot::{channel, Receiver, Sender};
 
 use super::{HummockError, HummockResult};
@@ -88,34 +88,14 @@ pub fn validate_safe_epoch(
     Ok(())
 }
 
-pub fn validate_table_key_range(version: &HummockVersion) {
-    for l in version.levels.values().flat_map(|levels| {
-        levels
-            .l0
-            .as_ref()
-            .unwrap()
-            .sub_levels
-            .iter()
-            .chain(levels.levels.iter())
-    }) {
-        for t in &l.table_infos {
-            assert!(
-                t.key_range.is_some(),
-                "key_range in table [{}] is none",
-                t.get_object_id()
-            );
-        }
-    }
-}
-
 pub fn filter_single_sst<R, B>(info: &SstableInfo, table_id: TableId, table_key_range: &R) -> bool
 where
     R: RangeBounds<TableKey<B>>,
     B: AsRef<[u8]> + EmptySliceRef,
 {
-    let table_range = info.key_range.as_ref().unwrap();
-    let table_start = FullKey::decode(table_range.left.as_slice()).user_key;
-    let table_end = FullKey::decode(table_range.right.as_slice()).user_key;
+    let table_range = &info.key_range;
+    let table_start = FullKey::decode(table_range.left.as_ref()).user_key;
+    let table_end = FullKey::decode(table_range.right.as_ref()).user_key;
     let (left, right) = bound_table_key_range(table_id, table_key_range);
     let left: Bound<UserKey<&[u8]>> = left.as_ref().map(|key| key.as_ref());
     let right: Bound<UserKey<&[u8]>> = right.as_ref().map(|key| key.as_ref());
@@ -127,18 +107,13 @@ where
         } else {
             Bound::Included(&table_end)
         },
-    ) && info
-        .get_table_ids()
-        .binary_search(&table_id.table_id())
-        .is_ok()
+    ) && info.table_ids.binary_search(&table_id.table_id()).is_ok()
 }
 
 /// Search the SST containing the specified key within a level, using binary search.
 pub(crate) fn search_sst_idx(ssts: &[SstableInfo], key: UserKey<&[u8]>) -> usize {
     ssts.partition_point(|table| {
-        let ord = FullKey::decode(&table.key_range.as_ref().unwrap().left)
-            .user_key
-            .cmp(&key);
+        let ord = FullKey::decode(&table.key_range.left).user_key.cmp(&key);
         ord == Ordering::Less || ord == Ordering::Equal
     })
 }
diff --git a/src/storage/src/hummock/vacuum.rs b/src/storage/src/hummock/vacuum.rs
index 5242a6eae0784..fb4c9d782215b 100644
--- a/src/storage/src/hummock/vacuum.rs
+++ b/src/storage/src/hummock/vacuum.rs
@@ -99,11 +99,12 @@ impl Vacuum {
         sstable_store: SstableStoreRef,
     ) -> HummockResult<(Vec<HummockSstableObjectId>, u64, u64)> {
         tracing::info!(
-            "Try to full scan SSTs with timestamp {}",
-            full_scan_task.sst_retention_time_sec
+            timestamp = full_scan_task.sst_retention_time_sec,
+            prefix = full_scan_task.prefix.as_ref().unwrap_or(&String::from("")),
+            "Try to full scan SSTs"
         );
         let metadata_iter = sstable_store
-            .list_object_metadata_from_object_store()
+            .list_object_metadata_from_object_store(full_scan_task.prefix.clone())
             .await?;
         Vacuum::full_scan_inner(full_scan_task, metadata_iter).await
     }
diff --git a/src/storage/src/hummock/validator.rs b/src/storage/src/hummock/validator.rs
index 8c38a99f4324b..cc95b7089b664 100644
--- a/src/storage/src/hummock/validator.rs
+++ b/src/storage/src/hummock/validator.rs
@@ -17,8 +17,8 @@ use std::cmp;
 use std::collections::HashMap;
 use std::sync::Arc;
 
+use risingwave_hummock_sdk::compact_task::ValidationTask;
 use risingwave_hummock_sdk::key::FullKey;
-use risingwave_pb::hummock::ValidationTask;
 
 use crate::hummock::iterator::HummockIterator;
 use crate::hummock::sstable::SstableIteratorReadOptions;
@@ -40,7 +40,7 @@ pub async fn validate_ssts(task: ValidationTask, sstable_store: SstableStoreRef)
             .expect("valid worker_id");
         tracing::debug!(
             "Validating SST {} from worker {}, epoch {}",
-            sst.get_object_id(),
+            sst.object_id,
             worker_id,
             task.epoch
         );
@@ -48,7 +48,7 @@ pub async fn validate_ssts(task: ValidationTask, sstable_store: SstableStoreRef)
             Ok(holder) => holder,
             Err(_err) => {
                 // One reasonable cause is the SST has been vacuumed.
-                tracing::info!("Skip sanity check for SST {}.", sst.get_object_id());
+                tracing::info!("Skip sanity check for SST {}.", sst.object_id);
                 continue;
             }
         };
@@ -67,7 +67,7 @@ pub async fn validate_ssts(task: ValidationTask, sstable_store: SstableStoreRef)
         );
         let mut previous_key: Option<FullKey<Vec<u8>>> = None;
         if let Err(_err) = iter.rewind().await {
-            tracing::info!("Skip sanity check for SST {}.", sst.get_object_id());
+            tracing::info!("Skip sanity check for SST {}.", sst.object_id);
         }
         while iter.is_valid() {
             key_counts += 1;
@@ -78,38 +78,32 @@ pub async fn validate_ssts(task: ValidationTask, sstable_store: SstableStoreRef)
             {
                 panic!("SST sanity check failed: Duplicate key {:x?} in SST object {} from worker {} and SST object {} from worker {}",
                        current_key,
-                       sst.get_object_id(),
+                       sst.object_id,
                        worker_id,
                        duplicate_sst_object_id,
                        duplicate_worker_id)
             }
-            visited_keys.insert(current_key.to_owned(), (sst.get_object_id(), worker_id));
+            visited_keys.insert(current_key.to_owned(), (sst.object_id, worker_id));
             // Ordered and Locally unique
             if let Some(previous_key) = previous_key.take() {
                 let cmp = previous_key.cmp(&current_key);
                 if cmp != cmp::Ordering::Less {
                     panic!(
                         "SST sanity check failed: For SST {}, expect {:x?} < {:x?}, got {:#?}",
-                        sst.get_object_id(),
-                        previous_key,
-                        current_key,
-                        cmp
+                        sst.object_id, previous_key, current_key, cmp
                     )
                 }
             }
             previous_key = Some(current_key);
             if let Err(_err) = iter.next().await {
-                tracing::info!(
-                    "Skip remaining sanity check for SST {}",
-                    sst.get_object_id(),
-                );
+                tracing::info!("Skip remaining sanity check for SST {}", sst.object_id,);
                 break;
             }
         }
         tracing::debug!(
             "Validated {} keys for SST {},  epoch {}",
             key_counts,
-            sst.get_object_id(),
+            sst.object_id,
             task.epoch
         );
         iter.collect_local_statistic(&mut unused);
diff --git a/src/storage/src/lib.rs b/src/storage/src/lib.rs
index 21c0c7f49ae4c..e11d3e1cee1ca 100644
--- a/src/storage/src/lib.rs
+++ b/src/storage/src/lib.rs
@@ -31,15 +31,14 @@
 #![feature(is_sorted)]
 #![feature(btree_extract_if)]
 #![feature(exact_size_is_empty)]
-#![feature(lazy_cell)]
 #![cfg_attr(coverage, feature(coverage_attribute))]
 #![recursion_limit = "256"]
 #![feature(error_generic_member_access)]
 #![feature(let_chains)]
-#![feature(exclusive_range_pattern)]
 #![feature(impl_trait_in_assoc_type)]
 #![feature(maybe_uninit_uninit_array)]
 #![feature(maybe_uninit_array_assume_init)]
+#![feature(iter_from_coroutine)]
 
 pub mod hummock;
 pub mod memory;
diff --git a/src/storage/src/mem_table.rs b/src/storage/src/mem_table.rs
index f7c5073a6a8cf..50984052fc5bc 100644
--- a/src/storage/src/mem_table.rs
+++ b/src/storage/src/mem_table.rs
@@ -24,9 +24,9 @@ use bytes::Bytes;
 use futures::{pin_mut, Stream, StreamExt};
 use futures_async_stream::try_stream;
 use itertools::Itertools;
-use risingwave_common::buffer::Bitmap;
+use risingwave_common::bitmap::Bitmap;
 use risingwave_common::catalog::{TableId, TableOption};
-use risingwave_common::hash::VnodeBitmapExt;
+use risingwave_common::hash::{VirtualNode, VnodeBitmapExt};
 use risingwave_common_estimate_size::{EstimateSize, KvSize};
 use risingwave_hummock_sdk::key::{prefixed_range_with_vnode, FullKey, TableKey, TableKeyRange};
 use risingwave_hummock_sdk::table_watermark::WatermarkDirection;
@@ -66,7 +66,7 @@ pub struct MemTable {
 
 #[derive(Error, Debug)]
 pub enum MemTableError {
-    #[error("Inconsistent operation")]
+    #[error("Inconsistent operation {key:?}, prev: {prev:?}, new: {new:?}")]
     InconsistentOperation {
         key: TableKey<Bytes>,
         prev: KeyOp,
@@ -546,15 +546,6 @@ impl<S: StateStoreWrite + StateStoreRead> LocalStateStore for MemtableLocalState
     type Iter<'a> = impl StateStoreIter + 'a;
     type RevIter<'a> = impl StateStoreIter + 'a;
 
-    #[allow(clippy::unused_async)]
-    async fn may_exist(
-        &self,
-        _key_range: TableKeyRange,
-        _read_options: ReadOptions,
-    ) -> StorageResult<bool> {
-        Ok(true)
-    }
-
     async fn get(
         &self,
         key: TableKey<Bytes>,
@@ -770,6 +761,11 @@ impl<S: StateStoreWrite + StateStoreRead> LocalStateStore for MemtableLocalState
     fn update_vnode_bitmap(&mut self, vnodes: Arc<Bitmap>) -> Arc<Bitmap> {
         std::mem::replace(&mut self.vnodes, vnodes)
     }
+
+    fn get_table_watermark(&self, _vnode: VirtualNode) -> Option<Bytes> {
+        // TODO: may store the written table watermark and have a correct implementation
+        None
+    }
 }
 
 #[cfg(test)]
diff --git a/src/storage/src/monitor/compactor_metrics.rs b/src/storage/src/monitor/compactor_metrics.rs
index 6c1d651ceddf3..c6289cdf77f39 100644
--- a/src/storage/src/monitor/compactor_metrics.rs
+++ b/src/storage/src/monitor/compactor_metrics.rs
@@ -50,6 +50,7 @@ pub struct CompactorMetrics {
     pub sstable_distinct_epoch_count: Histogram,
     pub compaction_event_consumed_latency: Histogram,
     pub compaction_event_loop_iteration_latency: Histogram,
+    pub sstable_block_size: Histogram,
 }
 
 pub static GLOBAL_COMPACTOR_METRICS: LazyLock<CompactorMetrics> =
@@ -194,7 +195,7 @@ impl CompactorMetrics {
         let opts = histogram_opts!(
             "compactor_sstable_avg_value_size",
             "Total bytes gotten from sstable_avg_value_size, for observing sstable_avg_value_size",
-            size_buckets
+            size_buckets.clone()
         );
 
         let sstable_avg_value_size = register_histogram_with_registry!(opts, registry).unwrap();
@@ -251,6 +252,14 @@ impl CompactorMetrics {
         let compaction_event_loop_iteration_latency =
             register_histogram_with_registry!(opts, registry).unwrap();
 
+        let opts = histogram_opts!(
+            "compactor_sstable_block_size",
+            "Total bytes gotten from sstable_block_size, for observing sstable_block_size",
+            size_buckets,
+        );
+
+        let sstable_block_size = register_histogram_with_registry!(opts, registry).unwrap();
+
         Self {
             compaction_upload_sst_counts,
             compact_fast_runner_bytes,
@@ -277,6 +286,7 @@ impl CompactorMetrics {
             sstable_distinct_epoch_count,
             compaction_event_consumed_latency,
             compaction_event_loop_iteration_latency,
+            sstable_block_size,
         }
     }
 
diff --git a/src/storage/src/monitor/local_metrics.rs b/src/storage/src/monitor/local_metrics.rs
index c218e484265bf..81bfb86fe3c73 100644
--- a/src/storage/src/monitor/local_metrics.rs
+++ b/src/storage/src/monitor/local_metrics.rs
@@ -51,7 +51,6 @@ pub struct StoreLocalStatistic {
     pub staging_sst_iter_count: u64,
     pub overlapping_iter_count: u64,
     pub non_overlapping_iter_count: u64,
-    pub may_exist_check_sstable_count: u64,
     pub sub_iter_count: u64,
     pub found_key: bool,
 
@@ -107,6 +106,13 @@ impl StoreLocalStatistic {
         }
     }
 
+    pub fn discard(self) {
+        #[cfg(all(debug_assertions, not(any(madsim, test, feature = "test"))))]
+        {
+            self.reported.fetch_or(true, Ordering::Relaxed);
+        }
+    }
+
     pub fn report_compactor(&self, metrics: &CompactorMetrics) {
         let t = self.remote_io_time.load(Ordering::Relaxed) as f64;
         if t > 0.0 {
@@ -233,11 +239,9 @@ struct LocalStoreMetrics {
     staging_sst_iter_count: LocalHistogram,
     overlapping_iter_count: LocalHistogram,
     non_overlapping_iter_count: LocalHistogram,
-    may_exist_check_sstable_count: LocalHistogram,
     sub_iter_count: LocalHistogram,
     iter_filter_metrics: BloomFilterLocalMetrics,
     get_filter_metrics: BloomFilterLocalMetrics,
-    may_exist_filter_metrics: BloomFilterLocalMetrics,
     collect_count: usize,
 
     staging_imm_get_count: LocalHistogram,
@@ -324,18 +328,12 @@ impl LocalStoreMetrics {
             .iter_merge_sstable_counts
             .with_label_values(&[table_id_label, "committed-non-overlapping-iter"])
             .local();
-        let may_exist_check_sstable_count = metrics
-            .iter_merge_sstable_counts
-            .with_label_values(&[table_id_label, "may-exist-check-sstable"])
-            .local();
         let sub_iter_count = metrics
             .iter_merge_sstable_counts
             .with_label_values(&[table_id_label, "sub-iter"])
             .local();
         let get_filter_metrics = BloomFilterLocalMetrics::new(metrics, table_id_label, "get");
         let iter_filter_metrics = BloomFilterLocalMetrics::new(metrics, table_id_label, "iter");
-        let may_exist_filter_metrics =
-            BloomFilterLocalMetrics::new(metrics, table_id_label, "may_exist");
 
         let staging_imm_get_count = metrics
             .iter_merge_sstable_counts
@@ -372,10 +370,8 @@ impl LocalStoreMetrics {
             overlapping_iter_count,
             sub_iter_count,
             non_overlapping_iter_count,
-            may_exist_check_sstable_count,
             get_filter_metrics,
             iter_filter_metrics,
-            may_exist_filter_metrics,
             collect_count: 0,
             staging_imm_get_count,
             staging_sst_get_count,
@@ -425,7 +421,6 @@ add_local_metrics_histogram!(
     overlapping_iter_count,
     non_overlapping_iter_count,
     sub_iter_count,
-    may_exist_check_sstable_count,
     staging_imm_get_count,
     staging_sst_get_count,
     overlapping_get_count,
@@ -574,37 +569,3 @@ impl Drop for IterLocalMetricsGuard {
         });
     }
 }
-
-pub struct MayExistLocalMetricsGuard {
-    metrics: Arc<HummockStateStoreMetrics>,
-    table_id: TableId,
-    pub local_stats: StoreLocalStatistic,
-}
-
-impl MayExistLocalMetricsGuard {
-    pub fn new(metrics: Arc<HummockStateStoreMetrics>, table_id: TableId) -> Self {
-        Self {
-            metrics,
-            table_id,
-            local_stats: StoreLocalStatistic::default(),
-        }
-    }
-}
-
-impl Drop for MayExistLocalMetricsGuard {
-    fn drop(&mut self) {
-        LOCAL_METRICS.with_borrow_mut(|local_metrics| {
-            let table_metrics = local_metrics
-                .entry(self.table_id.table_id)
-                .or_insert_with(|| {
-                    LocalStoreMetrics::new(
-                        self.metrics.as_ref(),
-                        self.table_id.to_string().as_str(),
-                    )
-                });
-            self.local_stats.report(table_metrics);
-            self.local_stats
-                .report_bloom_filter_metrics(&table_metrics.may_exist_filter_metrics);
-        });
-    }
-}
diff --git a/src/storage/src/monitor/monitored_storage_metrics.rs b/src/storage/src/monitor/monitored_storage_metrics.rs
index bbf904d38381a..01419586f2eea 100644
--- a/src/storage/src/monitor/monitored_storage_metrics.rs
+++ b/src/storage/src/monitor/monitored_storage_metrics.rs
@@ -18,14 +18,14 @@ use std::sync::{Arc, OnceLock};
 use std::time::{Duration, Instant};
 
 use prometheus::{
-    exponential_buckets, histogram_opts, linear_buckets, register_histogram_vec_with_registry,
-    register_histogram_with_registry, Histogram, Registry,
+    exponential_buckets, histogram_opts, linear_buckets, register_histogram_with_registry,
+    Histogram, Registry,
 };
 use risingwave_common::config::MetricLevel;
 use risingwave_common::metrics::{
     LabelGuardedIntCounterVec, LabelGuardedIntGauge, LabelGuardedLocalHistogram,
     LabelGuardedLocalIntCounter, RelabeledGuardedHistogramVec, RelabeledGuardedIntCounterVec,
-    RelabeledGuardedIntGaugeVec, RelabeledHistogramVec,
+    RelabeledGuardedIntGaugeVec,
 };
 use risingwave_common::monitor::GLOBAL_METRICS_REGISTRY;
 use risingwave_common::{
@@ -55,7 +55,6 @@ pub struct MonitoredStorageMetrics {
 
     // [table_id, op_type]
     pub iter_log_op_type_counts: LabelGuardedIntCounterVec<2>,
-    pub may_exist_duration: RelabeledHistogramVec,
 
     pub sync_duration: Histogram,
     pub sync_size: Histogram,
@@ -239,19 +238,6 @@ impl MonitoredStorageMetrics {
         )
         .unwrap();
 
-        let opts = histogram_opts!(
-            "state_store_may_exist_duration",
-            "Histogram of may exist time that have been issued to state store",
-            buckets,
-        );
-        let may_exist_duration =
-            register_histogram_vec_with_registry!(opts, &["table_id"], registry).unwrap();
-        let may_exist_duration = RelabeledHistogramVec::with_metric_level(
-            MetricLevel::Debug,
-            may_exist_duration,
-            metric_level,
-        );
-
         let opts = histogram_opts!(
             "state_store_sync_duration",
             "Histogram of time spent on compacting shared buffer to remote storage",
@@ -277,7 +263,6 @@ impl MonitoredStorageMetrics {
             iter_counts,
             iter_in_progress_counts,
             iter_log_op_type_counts,
-            may_exist_duration,
             sync_duration,
             sync_size,
         }
diff --git a/src/storage/src/monitor/monitored_store.rs b/src/storage/src/monitor/monitored_store.rs
index 95791714ef64a..8c00435541d2c 100644
--- a/src/storage/src/monitor/monitored_store.rs
+++ b/src/storage/src/monitor/monitored_store.rs
@@ -19,8 +19,9 @@ use std::sync::Arc;
 use await_tree::InstrumentAwait;
 use bytes::Bytes;
 use futures::{Future, TryFutureExt};
-use risingwave_common::buffer::Bitmap;
+use risingwave_common::bitmap::Bitmap;
 use risingwave_common::catalog::TableId;
+use risingwave_common::hash::VirtualNode;
 use risingwave_hummock_sdk::key::{TableKey, TableKeyRange};
 use risingwave_hummock_sdk::HummockReadEpoch;
 use thiserror_ext::AsReport;
@@ -205,26 +206,6 @@ impl<S: LocalStateStore> LocalStateStore for MonitoredStateStore<S> {
     type Iter<'a> = impl StateStoreIter + 'a;
     type RevIter<'a> = impl StateStoreIter + 'a;
 
-    async fn may_exist(
-        &self,
-        key_range: TableKeyRange,
-        read_options: ReadOptions,
-    ) -> StorageResult<bool> {
-        let table_id_label = read_options.table_id.to_string();
-        let timer = self
-            .storage_metrics
-            .may_exist_duration
-            .with_label_values(&[table_id_label.as_str()])
-            .start_timer();
-        let res = self
-            .inner
-            .may_exist(key_range, read_options)
-            .verbose_instrument_await("store_may_exist")
-            .await;
-        timer.observe_duration();
-        res
-    }
-
     fn get(
         &self,
         key: TableKey<Bytes>,
@@ -305,6 +286,10 @@ impl<S: LocalStateStore> LocalStateStore for MonitoredStateStore<S> {
     fn update_vnode_bitmap(&mut self, vnodes: Arc<Bitmap>) -> Arc<Bitmap> {
         self.inner.update_vnode_bitmap(vnodes)
     }
+
+    fn get_table_watermark(&self, vnode: VirtualNode) -> Option<Bytes> {
+        self.inner.get_table_watermark(vnode)
+    }
 }
 
 impl<S: StateStore> StateStore for MonitoredStateStore<S> {
diff --git a/src/storage/src/monitor/traced_store.rs b/src/storage/src/monitor/traced_store.rs
index d7b9bd66edcdd..b31c8fd0d73e8 100644
--- a/src/storage/src/monitor/traced_store.rs
+++ b/src/storage/src/monitor/traced_store.rs
@@ -17,8 +17,9 @@ use std::sync::Arc;
 
 use bytes::Bytes;
 use futures::{Future, FutureExt};
-use risingwave_common::buffer::Bitmap;
+use risingwave_common::bitmap::Bitmap;
 use risingwave_common::catalog::TableId;
+use risingwave_common::hash::VirtualNode;
 use risingwave_hummock_sdk::key::{TableKey, TableKeyRange};
 use risingwave_hummock_sdk::HummockReadEpoch;
 use risingwave_hummock_trace::{
@@ -110,14 +111,6 @@ impl<S: LocalStateStore> LocalStateStore for TracedStateStore<S> {
     type Iter<'a> = impl StateStoreIter + 'a;
     type RevIter<'a> = impl StateStoreIter + 'a;
 
-    fn may_exist(
-        &self,
-        key_range: TableKeyRange,
-        read_options: ReadOptions,
-    ) -> impl Future<Output = StorageResult<bool>> + Send + '_ {
-        self.inner.may_exist(key_range, read_options)
-    }
-
     fn get(
         &self,
         key: TableKey<Bytes>,
@@ -240,6 +233,10 @@ impl<S: LocalStateStore> LocalStateStore for TracedStateStore<S> {
     fn update_vnode_bitmap(&mut self, vnodes: Arc<Bitmap>) -> Arc<Bitmap> {
         self.inner.update_vnode_bitmap(vnodes)
     }
+
+    fn get_table_watermark(&self, vnode: VirtualNode) -> Option<Bytes> {
+        self.inner.get_table_watermark(vnode)
+    }
 }
 
 impl<S: StateStore> StateStore for TracedStateStore<S> {
diff --git a/src/storage/src/opts.rs b/src/storage/src/opts.rs
index 143b6bba37981..4c8d8ae8e2bb9 100644
--- a/src/storage/src/opts.rs
+++ b/src/storage/src/opts.rs
@@ -90,6 +90,7 @@ pub struct StorageOpts {
     pub data_file_cache_insert_rate_limit_mb: usize,
     pub data_file_cache_indexer_shards: usize,
     pub data_file_cache_compression: String,
+    pub data_file_cache_flush_buffer_threshold_mb: usize,
 
     pub cache_refill_data_refill_levels: Vec<u32>,
     pub cache_refill_timeout_ms: u64,
@@ -108,6 +109,7 @@ pub struct StorageOpts {
     pub meta_file_cache_insert_rate_limit_mb: usize,
     pub meta_file_cache_indexer_shards: usize,
     pub meta_file_cache_compression: String,
+    pub meta_file_cache_flush_buffer_threshold_mb: usize,
 
     /// The storage url for storing backups.
     pub backup_storage_url: String,
@@ -183,6 +185,7 @@ impl From<(&RwConfig, &SystemParamsReader, &StorageMemoryConfig)> for StorageOpt
             data_file_cache_insert_rate_limit_mb: c.storage.data_file_cache.insert_rate_limit_mb,
             data_file_cache_indexer_shards: c.storage.data_file_cache.indexer_shards,
             data_file_cache_compression: c.storage.data_file_cache.compression.clone(),
+            data_file_cache_flush_buffer_threshold_mb: s.block_file_cache_flush_buffer_threshold_mb,
             meta_file_cache_dir: c.storage.meta_file_cache.dir.clone(),
             meta_file_cache_capacity_mb: c.storage.meta_file_cache.capacity_mb,
             meta_file_cache_file_capacity_mb: c.storage.meta_file_cache.file_capacity_mb,
@@ -192,6 +195,7 @@ impl From<(&RwConfig, &SystemParamsReader, &StorageMemoryConfig)> for StorageOpt
             meta_file_cache_insert_rate_limit_mb: c.storage.meta_file_cache.insert_rate_limit_mb,
             meta_file_cache_indexer_shards: c.storage.meta_file_cache.indexer_shards,
             meta_file_cache_compression: c.storage.meta_file_cache.compression.clone(),
+            meta_file_cache_flush_buffer_threshold_mb: s.meta_file_cache_flush_buffer_threshold_mb,
             cache_refill_data_refill_levels: c.storage.cache_refill.data_refill_levels.clone(),
             cache_refill_timeout_ms: c.storage.cache_refill.timeout_ms,
             cache_refill_concurrency: c.storage.cache_refill.concurrency,
diff --git a/src/storage/src/panic_store.rs b/src/storage/src/panic_store.rs
index 831f1f09a1c56..a9e10c6553c54 100644
--- a/src/storage/src/panic_store.rs
+++ b/src/storage/src/panic_store.rs
@@ -18,8 +18,9 @@ use std::ops::Bound;
 use std::sync::Arc;
 
 use bytes::Bytes;
-use risingwave_common::buffer::Bitmap;
+use risingwave_common::bitmap::Bitmap;
 use risingwave_common::catalog::TableId;
+use risingwave_common::hash::VirtualNode;
 use risingwave_hummock_sdk::key::{TableKey, TableKeyRange};
 use risingwave_hummock_sdk::HummockReadEpoch;
 
@@ -92,15 +93,6 @@ impl LocalStateStore for PanicStateStore {
     type Iter<'a> = PanicStateStoreIter<StateStoreIterItem>;
     type RevIter<'a> = PanicStateStoreIter<StateStoreIterItem>;
 
-    #[allow(clippy::unused_async)]
-    async fn may_exist(
-        &self,
-        _key_range: TableKeyRange,
-        _read_options: ReadOptions,
-    ) -> StorageResult<bool> {
-        panic!("should not call may_exist from the state store!");
-    }
-
     #[allow(clippy::unused_async)]
     async fn get(
         &self,
@@ -171,6 +163,10 @@ impl LocalStateStore for PanicStateStore {
     fn update_vnode_bitmap(&mut self, _vnodes: Arc<Bitmap>) -> Arc<Bitmap> {
         panic!("should not operate on the panic state store!");
     }
+
+    fn get_table_watermark(&self, _vnode: VirtualNode) -> Option<Bytes> {
+        panic!("should not operate on the panic state store!");
+    }
 }
 
 impl StateStore for PanicStateStore {
diff --git a/src/storage/src/storage_failpoints/test_iterator.rs b/src/storage/src/storage_failpoints/test_iterator.rs
index 2a9fb64744371..c794c5c73bdce 100644
--- a/src/storage/src/storage_failpoints/test_iterator.rs
+++ b/src/storage/src/storage_failpoints/test_iterator.rs
@@ -403,7 +403,6 @@ async fn test_failpoints_compactor_iterator_recreate() {
         table.meta.block_metas.clone(),
         info,
         HashSet::from_iter(std::iter::once(0)),
-        0,
         &stats,
         Arc::new(TaskProgress::default()),
         sstable_store,
diff --git a/src/storage/src/store.rs b/src/storage/src/store.rs
index 9e373d3069bb1..4b837ce6d098e 100644
--- a/src/storage/src/store.rs
+++ b/src/storage/src/store.rs
@@ -25,7 +25,8 @@ use bytes::Bytes;
 use futures::{Stream, TryStreamExt};
 use futures_async_stream::try_stream;
 use prost::Message;
-use risingwave_common::buffer::Bitmap;
+use risingwave_common::array::Op;
+use risingwave_common::bitmap::Bitmap;
 use risingwave_common::catalog::{TableId, TableOption};
 use risingwave_common::hash::VirtualNode;
 use risingwave_common::util::epoch::{Epoch, EpochPair};
@@ -36,6 +37,7 @@ use risingwave_hummock_trace::{
     TracedInitOptions, TracedNewLocalOptions, TracedOpConsistencyLevel, TracedPrefetchOptions,
     TracedReadOptions, TracedSealCurrentEpochOptions, TracedWriteOptions,
 };
+use risingwave_pb::hummock::PbVnodeWatermark;
 
 use crate::error::{StorageError, StorageResult};
 use crate::hummock::CachePolicy;
@@ -189,6 +191,27 @@ impl<T> ChangeLogValue<T> {
             ChangeLogValue::Delete(value) => ChangeLogValue::Delete(f(value)?),
         })
     }
+
+    pub fn into_op_value_iter(self) -> impl Iterator<Item = (Op, T)> {
+        std::iter::from_coroutine(
+            #[coroutine]
+            move || match self {
+                Self::Insert(row) => {
+                    yield (Op::Insert, row);
+                }
+                Self::Delete(row) => {
+                    yield (Op::Delete, row);
+                }
+                Self::Update {
+                    old_value,
+                    new_value,
+                } => {
+                    yield (Op::UpdateDelete, old_value);
+                    yield (Op::UpdateInsert, new_value);
+                }
+            },
+        )
+    }
 }
 
 impl<T: AsRef<[u8]>> ChangeLogValue<T> {
@@ -383,6 +406,9 @@ pub trait LocalStateStore: StaticSendSync {
         read_options: ReadOptions,
     ) -> impl Future<Output = StorageResult<Self::RevIter<'_>>> + Send + '_;
 
+    /// Get last persisted watermark for a given vnode.
+    fn get_table_watermark(&self, vnode: VirtualNode) -> Option<Bytes>;
+
     /// Inserts a key-value entry associated with a given `epoch` into the state store.
     fn insert(
         &mut self,
@@ -415,21 +441,6 @@ pub trait LocalStateStore: StaticSendSync {
     /// the previous write epoch is sealed.
     fn seal_current_epoch(&mut self, next_epoch: u64, opts: SealCurrentEpochOptions);
 
-    /// Check existence of a given `key_range`.
-    /// It is better to provide `prefix_hint` in `read_options`, which will be used
-    /// for checking bloom filter if hummock is used. If `prefix_hint` is not provided,
-    /// the false positive rate can be significantly higher because bloom filter cannot
-    /// be used.
-    ///
-    /// Returns:
-    /// - false: `key_range` is guaranteed to be absent in storage.
-    /// - true: `key_range` may or may not exist in storage.
-    fn may_exist(
-        &self,
-        key_range: TableKeyRange,
-        read_options: ReadOptions,
-    ) -> impl Future<Output = StorageResult<bool>> + Send + '_;
-
     // Updates the vnode bitmap corresponding to the local state store
     // Returns the previous vnode bitmap
     fn update_vnode_bitmap(&mut self, vnodes: Arc<Bitmap>) -> Arc<Bitmap>;
@@ -501,6 +512,7 @@ pub struct ReadOptions {
     /// Read from historical hummock version of meta snapshot backup.
     /// It should only be used by `StorageTable` for batch query.
     pub read_version_from_backup: bool,
+    pub read_version_from_time_travel: bool,
 }
 
 impl From<TracedReadOptions> for ReadOptions {
@@ -513,6 +525,7 @@ impl From<TracedReadOptions> for ReadOptions {
             retention_seconds: value.retention_seconds,
             table_id: value.table_id.into(),
             read_version_from_backup: value.read_version_from_backup,
+            read_version_from_time_travel: value.read_version_from_time_travel,
         }
     }
 }
@@ -527,6 +540,7 @@ impl From<ReadOptions> for TracedReadOptions {
             retention_seconds: value.retention_seconds,
             table_id: value.table_id.into(),
             read_version_from_backup: value.read_version_from_backup,
+            read_version_from_time_travel: value.read_version_from_time_travel,
         }
     }
 }
@@ -768,8 +782,11 @@ impl From<SealCurrentEpochOptions> for TracedSealCurrentEpochOptions {
                 (
                     direction == WatermarkDirection::Ascending,
                     watermarks
-                        .iter()
-                        .map(|watermark| Message::encode_to_vec(&watermark.to_protobuf()))
+                        .into_iter()
+                        .map(|watermark| {
+                            let pb_watermark = PbVnodeWatermark::from(watermark);
+                            Message::encode_to_vec(&pb_watermark)
+                        })
                         .collect(),
                 )
             }),
@@ -791,10 +808,10 @@ impl From<TracedSealCurrentEpochOptions> for SealCurrentEpochOptions {
                         WatermarkDirection::Descending
                     },
                     watermarks
-                        .iter()
+                        .into_iter()
                         .map(|serialized_watermark| {
                             Message::decode(serialized_watermark.as_slice())
-                                .map(|pb| VnodeWatermark::from_protobuf(&pb))
+                                .map(|pb: PbVnodeWatermark| VnodeWatermark::from(pb))
                                 .expect("should not failed")
                         })
                         .collect(),
diff --git a/src/storage/src/store_impl.rs b/src/storage/src/store_impl.rs
index 3c8284cb55c90..3ee9e849dda4c 100644
--- a/src/storage/src/store_impl.rs
+++ b/src/storage/src/store_impl.rs
@@ -22,7 +22,7 @@ use foyer::{
     DirectFsDeviceOptionsBuilder, HybridCacheBuilder, RateLimitPicker, RuntimeConfigBuilder,
 };
 use risingwave_common::monitor::GLOBAL_METRICS_REGISTRY;
-use risingwave_common_service::observer_manager::RpcNotificationClient;
+use risingwave_common_service::RpcNotificationClient;
 use risingwave_hummock_sdk::HummockSstableObjectId;
 use risingwave_object_store::object::build_remote_object_store;
 
@@ -42,9 +42,27 @@ use crate::monitor::{
 use crate::opts::StorageOpts;
 use crate::StateStore;
 
-pub type HummockStorageType = impl StateStore + AsHummock;
-pub type MemoryStateStoreType = impl StateStore + AsHummock;
-pub type SledStateStoreType = impl StateStore + AsHummock;
+mod opaque_type {
+    use super::*;
+
+    pub type HummockStorageType = impl StateStore + AsHummock;
+    pub type MemoryStateStoreType = impl StateStore + AsHummock;
+    pub type SledStateStoreType = impl StateStore + AsHummock;
+
+    pub fn in_memory(state_store: MemoryStateStore) -> MemoryStateStoreType {
+        may_dynamic_dispatch(state_store)
+    }
+
+    pub fn hummock(state_store: HummockStorage) -> HummockStorageType {
+        may_dynamic_dispatch(may_verify(state_store))
+    }
+
+    pub fn sled(state_store: SledStateStore) -> SledStateStoreType {
+        may_dynamic_dispatch(state_store)
+    }
+}
+use opaque_type::{hummock, in_memory, sled};
+pub use opaque_type::{HummockStorageType, MemoryStateStoreType, SledStateStoreType};
 
 /// The type erased [`StateStore`].
 #[derive(Clone, EnumAsInner)]
@@ -114,7 +132,7 @@ impl StateStoreImpl {
         storage_metrics: Arc<MonitoredStorageMetrics>,
     ) -> Self {
         // The specific type of MemoryStateStoreType in deducted here.
-        Self::MemoryStateStore(may_dynamic_dispatch(state_store).monitored(storage_metrics))
+        Self::MemoryStateStore(in_memory(state_store).monitored(storage_metrics))
     }
 
     pub fn hummock(
@@ -122,16 +140,14 @@ impl StateStoreImpl {
         storage_metrics: Arc<MonitoredStorageMetrics>,
     ) -> Self {
         // The specific type of HummockStateStoreType in deducted here.
-        Self::HummockStateStore(
-            may_dynamic_dispatch(may_verify(state_store)).monitored(storage_metrics),
-        )
+        Self::HummockStateStore(hummock(state_store).monitored(storage_metrics))
     }
 
     pub fn sled(
         state_store: SledStateStore,
         storage_metrics: Arc<MonitoredStorageMetrics>,
     ) -> Self {
-        Self::SledStateStore(may_dynamic_dispatch(state_store).monitored(storage_metrics))
+        Self::SledStateStore(sled(state_store).monitored(storage_metrics))
     }
 
     pub fn shared_in_memory_store(storage_metrics: Arc<MonitoredStorageMetrics>) -> Self {
@@ -214,8 +230,9 @@ pub mod verify {
     use std::sync::Arc;
 
     use bytes::Bytes;
-    use risingwave_common::buffer::Bitmap;
+    use risingwave_common::bitmap::Bitmap;
     use risingwave_common::catalog::TableId;
+    use risingwave_common::hash::VirtualNode;
     use risingwave_hummock_sdk::key::{TableKey, TableKeyRange};
     use risingwave_hummock_sdk::HummockReadEpoch;
     use tracing::log::warn;
@@ -281,7 +298,7 @@ pub mod verify {
 
         // TODO: may avoid manual async fn when the bug of rust compiler is fixed. Currently it will
         // fail to compile.
-        #[allow(clippy::manual_async_fn)]
+        #[expect(clippy::manual_async_fn)]
         fn iter(
             &self,
             key_range: TableKeyRange,
@@ -303,7 +320,7 @@ pub mod verify {
             }
         }
 
-        #[allow(clippy::manual_async_fn)]
+        #[expect(clippy::manual_async_fn)]
         fn rev_iter(
             &self,
             key_range: TableKeyRange,
@@ -408,17 +425,6 @@ pub mod verify {
         type Iter<'a> = impl StateStoreIter + 'a;
         type RevIter<'a> = impl StateStoreIter + 'a;
 
-        // We don't verify `may_exist` across different state stores because
-        // the return value of `may_exist` is implementation specific and may not
-        // be consistent across different state store backends.
-        fn may_exist(
-            &self,
-            key_range: TableKeyRange,
-            read_options: ReadOptions,
-        ) -> impl Future<Output = StorageResult<bool>> + Send + '_ {
-            self.actual.may_exist(key_range, read_options)
-        }
-
         async fn get(
             &self,
             key: TableKey<Bytes>,
@@ -432,7 +438,7 @@ pub mod verify {
             actual
         }
 
-        #[allow(clippy::manual_async_fn)]
+        #[expect(clippy::manual_async_fn)]
         fn iter(
             &self,
             key_range: TableKeyRange,
@@ -453,7 +459,7 @@ pub mod verify {
             }
         }
 
-        #[allow(clippy::manual_async_fn)]
+        #[expect(clippy::manual_async_fn)]
         fn rev_iter(
             &self,
             key_range: TableKeyRange,
@@ -548,6 +554,14 @@ pub mod verify {
             }
             ret
         }
+
+        fn get_table_watermark(&self, vnode: VirtualNode) -> Option<Bytes> {
+            let ret = self.actual.get_table_watermark(vnode);
+            if let Some(expected) = &self.expected {
+                assert_eq!(ret, expected.get_table_watermark(vnode));
+            }
+            ret
+        }
     }
 
     impl<A: StateStore, E: StateStore> StateStore for VerifyStateStore<A, E> {
@@ -655,6 +669,7 @@ impl StateStoreImpl {
                     .with_indexer_shards(opts.meta_file_cache_indexer_shards)
                     .with_flushers(opts.meta_file_cache_flushers)
                     .with_reclaimers(opts.meta_file_cache_reclaimers)
+                    .with_buffer_threshold(opts.meta_file_cache_flush_buffer_threshold_mb * MB) // 128 MiB
                     .with_clean_region_threshold(
                         opts.meta_file_cache_reclaimers + opts.meta_file_cache_reclaimers / 2,
                     )
@@ -707,6 +722,7 @@ impl StateStoreImpl {
                     .with_indexer_shards(opts.data_file_cache_indexer_shards)
                     .with_flushers(opts.data_file_cache_flushers)
                     .with_reclaimers(opts.data_file_cache_reclaimers)
+                    .with_buffer_threshold(opts.data_file_cache_flush_buffer_threshold_mb * MB) // 128 MiB
                     .with_clean_region_threshold(
                         opts.data_file_cache_reclaimers + opts.data_file_cache_reclaimers / 2,
                     )
@@ -834,8 +850,9 @@ pub mod boxed_state_store {
     use dyn_clone::{clone_trait_object, DynClone};
     use futures::future::BoxFuture;
     use futures::FutureExt;
-    use risingwave_common::buffer::Bitmap;
+    use risingwave_common::bitmap::Bitmap;
     use risingwave_common::catalog::TableId;
+    use risingwave_common::hash::VirtualNode;
     use risingwave_hummock_sdk::key::{TableKey, TableKeyRange};
     use risingwave_hummock_sdk::{HummockReadEpoch, SyncResult};
 
@@ -948,12 +965,6 @@ pub mod boxed_state_store {
     pub type BoxLocalStateStoreIterStream<'a> = BoxStateStoreIter<'a, StateStoreIterItem>;
     #[async_trait::async_trait]
     pub trait DynamicDispatchedLocalStateStore: StaticSendSync {
-        async fn may_exist(
-            &self,
-            key_range: TableKeyRange,
-            read_options: ReadOptions,
-        ) -> StorageResult<bool>;
-
         async fn get(
             &self,
             key: TableKey<Bytes>,
@@ -994,18 +1005,12 @@ pub mod boxed_state_store {
         fn seal_current_epoch(&mut self, next_epoch: u64, opts: SealCurrentEpochOptions);
 
         fn update_vnode_bitmap(&mut self, vnodes: Arc<Bitmap>) -> Arc<Bitmap>;
+
+        fn get_table_watermark(&self, vnode: VirtualNode) -> Option<Bytes>;
     }
 
     #[async_trait::async_trait]
     impl<S: LocalStateStore> DynamicDispatchedLocalStateStore for S {
-        async fn may_exist(
-            &self,
-            key_range: TableKeyRange,
-            read_options: ReadOptions,
-        ) -> StorageResult<bool> {
-            self.may_exist(key_range, read_options).await
-        }
-
         async fn get(
             &self,
             key: TableKey<Bytes>,
@@ -1070,6 +1075,10 @@ pub mod boxed_state_store {
         fn update_vnode_bitmap(&mut self, vnodes: Arc<Bitmap>) -> Arc<Bitmap> {
             self.update_vnode_bitmap(vnodes)
         }
+
+        fn get_table_watermark(&self, vnode: VirtualNode) -> Option<Bytes> {
+            self.get_table_watermark(vnode)
+        }
     }
 
     pub type BoxDynamicDispatchedLocalStateStore = Box<dyn DynamicDispatchedLocalStateStore>;
@@ -1078,14 +1087,6 @@ pub mod boxed_state_store {
         type Iter<'a> = BoxLocalStateStoreIterStream<'a>;
         type RevIter<'a> = BoxLocalStateStoreIterStream<'a>;
 
-        fn may_exist(
-            &self,
-            key_range: TableKeyRange,
-            read_options: ReadOptions,
-        ) -> impl Future<Output = StorageResult<bool>> + Send + '_ {
-            self.deref().may_exist(key_range, read_options)
-        }
-
         fn get(
             &self,
             key: TableKey<Bytes>,
@@ -1110,6 +1111,10 @@ pub mod boxed_state_store {
             self.deref().rev_iter(key_range, read_options)
         }
 
+        fn get_table_watermark(&self, vnode: VirtualNode) -> Option<Bytes> {
+            self.deref().get_table_watermark(vnode)
+        }
+
         fn insert(
             &mut self,
             key: TableKey<Bytes>,
diff --git a/src/storage/src/table/batch_table/storage_table.rs b/src/storage/src/table/batch_table/storage_table.rs
index e5f2801b05022..87b5006c0fa86 100644
--- a/src/storage/src/table/batch_table/storage_table.rs
+++ b/src/storage/src/table/batch_table/storage_table.rs
@@ -25,8 +25,8 @@ use futures::{Stream, StreamExt};
 use futures_async_stream::try_stream;
 use itertools::{Either, Itertools};
 use more_asserts::assert_gt;
-use risingwave_common::array::{ArrayBuilderImpl, ArrayRef, DataChunk, Op};
-use risingwave_common::buffer::Bitmap;
+use risingwave_common::array::{ArrayBuilderImpl, ArrayRef, DataChunk};
+use risingwave_common::bitmap::Bitmap;
 use risingwave_common::catalog::{ColumnDesc, ColumnId, Schema, TableId, TableOption};
 use risingwave_common::hash::{VirtualNode, VnodeBitmapExt};
 use risingwave_common::row::{self, OwnedRow, Row, RowExt};
@@ -47,9 +47,11 @@ use crate::hummock::CachePolicy;
 use crate::row_serde::row_serde_util::{serialize_pk, serialize_pk_with_vnode};
 use crate::row_serde::value_serde::{ValueRowSerde, ValueRowSerdeNew};
 use crate::row_serde::{find_columns_by_ids, ColumnMapping};
-use crate::store::{ChangeLogValue, PrefetchOptions, ReadLogOptions, ReadOptions, StateStoreIter};
+use crate::store::{
+    PrefetchOptions, ReadLogOptions, ReadOptions, StateStoreIter, StateStoreIterExt,
+};
 use crate::table::merge_sort::merge_sort;
-use crate::table::{KeyedRow, TableDistribution, TableIter};
+use crate::table::{ChangeLogRow, KeyedRow, TableDistribution, TableIter};
 use crate::StateStore;
 
 /// [`StorageTableInner`] is the interface accessing relational data in KV(`StateStore`) with
@@ -359,6 +361,7 @@ impl<S: StateStore, SD: ValueRowSerde> StorageTableInner<S, SD> {
     ) -> StorageResult<Option<OwnedRow>> {
         let epoch = wait_epoch.get_epoch();
         let read_backup = matches!(wait_epoch, HummockReadEpoch::Backup(_));
+        let read_time_travel = matches!(wait_epoch, HummockReadEpoch::TimeTravel(_));
         self.store.try_wait_epoch(wait_epoch).await?;
         let serialized_pk = serialize_pk_with_vnode(
             &pk,
@@ -379,6 +382,7 @@ impl<S: StateStore, SD: ValueRowSerde> StorageTableInner<S, SD> {
             retention_seconds: self.table_option.retention_seconds,
             table_id: self.table_id,
             read_version_from_backup: read_backup,
+            read_version_from_time_travel: read_time_travel,
             cache_policy: CachePolicy::Fill(CacheContext::Default),
             ..Default::default()
         };
@@ -486,12 +490,14 @@ impl<S: StateStore, SD: ValueRowSerde> StorageTableInner<S, SD> {
         let iterators: Vec<_> = try_join_all(table_key_ranges.map(|table_key_range| {
             let prefix_hint = prefix_hint.clone();
             let read_backup = matches!(wait_epoch, HummockReadEpoch::Backup(_));
+            let read_time_travel = matches!(wait_epoch, HummockReadEpoch::TimeTravel(_));
             async move {
                 let read_options = ReadOptions {
                     prefix_hint,
                     retention_seconds: self.table_option.retention_seconds,
                     table_id: self.table_id,
                     read_version_from_backup: read_backup,
+                    read_version_from_time_travel: read_time_travel,
                     prefetch_options,
                     cache_policy,
                     ..Default::default()
@@ -743,13 +749,12 @@ impl<S: StateStore, SD: ValueRowSerde> StorageTableInner<S, SD> {
 
     pub async fn batch_iter_log_with_pk_bounds(
         &self,
-        satrt_epoch: HummockReadEpoch,
-        end_epoch: HummockReadEpoch,
-        pk_prefix: impl Row,
-        range_bounds: impl RangeBounds<OwnedRow>,
-    ) -> StorageResult<impl Stream<Item = StorageResult<(Op, KeyedRow<Bytes>)>> + Send> {
-        let start_key = self.serialize_pk_bound(&pk_prefix, range_bounds.start_bound(), true);
-        let end_key = self.serialize_pk_bound(&pk_prefix, range_bounds.end_bound(), false);
+        start_epoch: u64,
+        end_epoch: u64,
+    ) -> StorageResult<impl Stream<Item = StorageResult<ChangeLogRow>> + Send> {
+        let pk_prefix = OwnedRow::default();
+        let start_key = self.serialize_pk_bound(&pk_prefix, Unbounded, true);
+        let end_key = self.serialize_pk_bound(&pk_prefix, Unbounded, false);
 
         assert!(pk_prefix.len() <= self.pk_indices.len());
         let table_key_ranges = {
@@ -774,7 +779,7 @@ impl<S: StateStore, SD: ValueRowSerde> StorageTableInner<S, SD> {
                 self.row_serde.clone(),
                 table_key_range,
                 read_options,
-                satrt_epoch,
+                start_epoch,
                 end_epoch,
             )
             .await?
@@ -969,18 +974,14 @@ impl<S: StateStore, SD: ValueRowSerde> StorageTableInnerIterLogInner<S, SD> {
         row_deserializer: Arc<SD>,
         table_key_range: TableKeyRange,
         read_options: ReadLogOptions,
-        satrt_epoch: HummockReadEpoch,
-        end_epoch: HummockReadEpoch,
+        start_epoch: u64,
+        end_epoch: u64,
     ) -> StorageResult<Self> {
-        let raw_satrt_epoch = satrt_epoch.get_epoch();
-        let raw_end_epoch = end_epoch.get_epoch();
-        store.try_wait_epoch(end_epoch).await?;
+        store
+            .try_wait_epoch(HummockReadEpoch::Committed(end_epoch))
+            .await?;
         let iter = store
-            .iter_log(
-                (raw_satrt_epoch, raw_end_epoch),
-                table_key_range,
-                read_options,
-            )
+            .iter_log((start_epoch, end_epoch), table_key_range, read_options)
             .await?;
         let iter = Self {
             iter,
@@ -991,77 +992,16 @@ impl<S: StateStore, SD: ValueRowSerde> StorageTableInnerIterLogInner<S, SD> {
     }
 
     /// Yield a row with its primary key.
-    #[try_stream(ok = (Op, KeyedRow<Bytes>), error = StorageError)]
-    async fn into_stream(mut self) {
-        while let Some((k, v)) = self
-            .iter
-            .try_next()
-            .verbose_instrument_await("storage_table_iter_next")
-            .await?
-        {
-            match v {
-                ChangeLogValue::Insert(value) => {
-                    let full_row = self.row_deserializer.deserialize(value)?;
-                    let row = self
-                        .mapping
-                        .project(OwnedRow::new(full_row))
-                        .into_owned_row();
-                    // TODO: may optimize the key clone
-                    yield (
-                        Op::Insert,
-                        KeyedRow::<Bytes> {
-                            vnode_prefixed_key: k.copy_into(),
-                            row,
-                        },
-                    );
-                }
-                ChangeLogValue::Update {
-                    new_value,
-                    old_value,
-                } => {
-                    let full_row = self.row_deserializer.deserialize(old_value)?;
-                    let row = self
-                        .mapping
-                        .project(OwnedRow::new(full_row))
-                        .into_owned_row();
-                    // TODO: may optimize the key clone
-                    yield (
-                        Op::UpdateDelete,
-                        KeyedRow::<Bytes> {
-                            vnode_prefixed_key: k.copy_into(),
-                            row,
-                        },
-                    );
-                    let full_row = self.row_deserializer.deserialize(new_value)?;
-                    let row = self
-                        .mapping
-                        .project(OwnedRow::new(full_row))
-                        .into_owned_row();
-                    // TODO: may optimize the key clone
-                    yield (
-                        Op::UpdateInsert,
-                        KeyedRow::<Bytes> {
-                            vnode_prefixed_key: k.copy_into(),
-                            row,
-                        },
-                    );
-                }
-                ChangeLogValue::Delete(value) => {
-                    let full_row = self.row_deserializer.deserialize(value)?;
-                    let row = self
-                        .mapping
-                        .project(OwnedRow::new(full_row))
-                        .into_owned_row();
-                    // TODO: may optimize the key clone
-                    yield (
-                        Op::Delete,
-                        KeyedRow::<Bytes> {
-                            vnode_prefixed_key: k.copy_into(),
-                            row,
-                        },
-                    );
-                }
-            }
-        }
+    fn into_stream(self) -> impl Stream<Item = StorageResult<ChangeLogRow>> {
+        self.iter.into_stream(move |(_key, value)| {
+            value.try_map(|value| {
+                let full_row = self.row_deserializer.deserialize(value)?;
+                let row = self
+                    .mapping
+                    .project(OwnedRow::new(full_row))
+                    .into_owned_row();
+                Ok(row)
+            })
+        })
     }
 }
diff --git a/src/storage/src/table/mod.rs b/src/storage/src/table/mod.rs
index d245e4bde3790..ceabcd14ea2d6 100644
--- a/src/storage/src/table/mod.rs
+++ b/src/storage/src/table/mod.rs
@@ -17,19 +17,17 @@ pub mod merge_sort;
 
 use std::ops::Deref;
 
-use bytes::Bytes;
 use futures::{Stream, StreamExt};
-use futures_async_stream::try_stream;
-use risingwave_common::array::{DataChunk, Op};
+use risingwave_common::array::DataChunk;
 use risingwave_common::catalog::Schema;
 pub use risingwave_common::hash::table_distribution::*;
 use risingwave_common::hash::VirtualNode;
 use risingwave_common::row::{OwnedRow, Row};
 use risingwave_common::util::chunk_coalesce::DataChunkBuilder;
-use risingwave_common::util::iter_util::ZipEqFast;
+use risingwave_common::util::iter_util::ZipEqDebug;
 use risingwave_hummock_sdk::key::TableKey;
 
-use crate::error::{StorageError, StorageResult};
+use crate::error::StorageResult;
 use crate::row_serde::value_serde::ValueRowSerde;
 use crate::store::{ChangeLogValue, StateStoreIterExt, StateStoreReadLogItem};
 use crate::StateStoreIter;
@@ -40,20 +38,21 @@ pub trait TableIter: Send {
     async fn next_row(&mut self) -> StorageResult<Option<OwnedRow>>;
 }
 
-pub async fn collect_data_chunk<E, S>(
+pub async fn collect_data_chunk<E, S, R>(
     stream: &mut S,
     schema: &Schema,
     chunk_size: Option<usize>,
 ) -> Result<Option<DataChunk>, E>
 where
-    S: Stream<Item = Result<KeyedRow<Bytes>, E>> + Unpin,
+    S: Stream<Item = Result<R, E>> + Unpin,
+    R: Row,
 {
     let mut builders = schema.create_array_builders(chunk_size.unwrap_or(0));
     let mut row_count = 0;
     for _ in 0..chunk_size.unwrap_or(usize::MAX) {
         match stream.next().await.transpose()? {
             Some(row) => {
-                for (datum, builder) in row.iter().zip_eq_fast(builders.iter_mut()) {
+                for (datum, builder) in row.iter().zip_eq_debug(builders.iter_mut()) {
                     builder.append(datum);
                 }
             }
@@ -151,30 +150,13 @@ impl<T: AsRef<[u8]>> Deref for KeyedRow<T> {
     }
 }
 
-#[try_stream(ok = (Op, OwnedRow), error = StorageError)]
-pub async fn deserialize_log_stream<'a>(
+pub type ChangeLogRow = ChangeLogValue<OwnedRow>;
+
+pub fn deserialize_log_stream<'a>(
     iter: impl StateStoreIter<StateStoreReadLogItem> + 'a,
     deserializer: &'a impl ValueRowSerde,
-) {
-    let stream = iter.into_stream(|(_key, log_value)| {
+) -> impl Stream<Item = StorageResult<ChangeLogRow>> + 'a {
+    iter.into_stream(|(_key, log_value)| {
         log_value.try_map(|slice| Ok(OwnedRow::new(deserializer.deserialize(slice)?)))
-    });
-    #[for_await]
-    for log_value in stream {
-        match log_value? {
-            ChangeLogValue::Insert(row) => {
-                yield (Op::Insert, row);
-            }
-            ChangeLogValue::Delete(row) => {
-                yield (Op::Delete, row);
-            }
-            ChangeLogValue::Update {
-                new_value,
-                old_value,
-            } => {
-                yield (Op::UpdateDelete, old_value);
-                yield (Op::UpdateInsert, new_value);
-            }
-        }
-    }
+    })
 }
diff --git a/src/stream/Cargo.toml b/src/stream/Cargo.toml
index 89f2b61fd3449..3c85092a4d677 100644
--- a/src/stream/Cargo.toml
+++ b/src/stream/Cargo.toml
@@ -32,12 +32,13 @@ foyer = { workspace = true }
 futures = { version = "0.3", default-features = false, features = ["alloc"] }
 futures-async-stream = { workspace = true }
 governor = { version = "0.6", default-features = false, features = [
-    "std",
-    "dashmap",
-    "jitter",
+  "std",
+  "dashmap",
+  "jitter",
 ] }
 hytra = "0.1.2"
 itertools = { workspace = true }
+jsonbb = { workspace = true }
 local_stats_alloc = { path = "../utils/local_stats_alloc" }
 lru = { workspace = true }
 maplit = "1.0.2"
@@ -68,13 +69,13 @@ strum_macros = "0.26"
 thiserror = "1"
 thiserror-ext = { workspace = true }
 tokio = { version = "0.2", package = "madsim-tokio", features = [
-    "rt",
-    "rt-multi-thread",
-    "sync",
-    "macros",
-    "time",
-    "signal",
-    "fs",
+  "rt",
+  "rt-multi-thread",
+  "sync",
+  "macros",
+  "time",
+  "signal",
+  "fs",
 ] }
 tokio-metrics = "0.3.0"
 tokio-retry = "0.3"
@@ -92,7 +93,7 @@ expect-test = "1"
 risingwave_expr_impl = { workspace = true }
 risingwave_hummock_sdk = { workspace = true }
 risingwave_hummock_test = { path = "../storage/hummock_test", features = [
-    "test",
+  "test",
 ] }
 serde_yaml = "0.9"
 tracing-test = "0.2"
diff --git a/src/stream/src/cache/mod.rs b/src/stream/src/cache/mod.rs
index 292500728f832..3f41f9134b3c0 100644
--- a/src/stream/src/cache/mod.rs
+++ b/src/stream/src/cache/mod.rs
@@ -14,7 +14,7 @@
 
 mod managed_lru;
 pub use managed_lru::*;
-use risingwave_common::buffer::Bitmap;
+use risingwave_common::bitmap::Bitmap;
 use risingwave_common::util::iter_util::ZipEqFast;
 
 /// Returns whether we're unsure about the fressness of the cache after the scaling from the
@@ -38,7 +38,7 @@ use risingwave_common::util::iter_util::ZipEqFast;
 ///
 /// This brings a problem when scaling in after a while. Some partitions may be reassigned back to
 /// the current executor, while the cache entries of these partitions are still unevicted. So it's
-/// possible that these entries have been updated by other executors on other parallel units, and
+/// possible that these entries have been updated by other executors on other workers, and
 /// the content is now stale! The executor must evict these entries which are not in the
 /// **previous** partition before further processing.
 pub(super) fn cache_may_stale(
diff --git a/src/stream/src/common/cache/top_n_cache.rs b/src/stream/src/common/cache/top_n_cache.rs
deleted file mode 100644
index e6e65bacea5fe..0000000000000
--- a/src/stream/src/common/cache/top_n_cache.rs
+++ /dev/null
@@ -1,191 +0,0 @@
-// Copyright 2024 RisingWave Labs
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::collections::BTreeMap;
-
-use risingwave_common_estimate_size::{EstimateSize, KvSize};
-
-/// Inner top-N cache structure for [`super::TopNStateCache`].
-#[derive(Clone)]
-pub struct TopNCache<K: Ord + EstimateSize, V: EstimateSize> {
-    /// The capacity of the cache.
-    capacity: usize,
-    /// Ordered cache entries.
-    entries: BTreeMap<K, V>,
-    kv_heap_size: KvSize,
-}
-
-impl<K: Ord + EstimateSize, V: EstimateSize> EstimateSize for TopNCache<K, V> {
-    fn estimated_heap_size(&self) -> usize {
-        // TODO: Add btreemap internal size.
-        // https://github.com/risingwavelabs/risingwave/issues/9713
-        self.kv_heap_size.size()
-    }
-}
-
-impl<K: Ord + EstimateSize, V: EstimateSize> TopNCache<K, V> {
-    /// Create a new cache with specified capacity and order requirements.
-    /// To create a cache with unlimited capacity, use `usize::MAX` for `capacity`.
-    pub fn new(capacity: usize) -> Self {
-        Self {
-            capacity,
-            entries: Default::default(),
-            kv_heap_size: KvSize::new(),
-        }
-    }
-
-    /// Get the capacity of the cache.
-    pub fn capacity(&self) -> usize {
-        self.capacity
-    }
-
-    /// Get the number of entries in the cache.
-    #[allow(dead_code)]
-    pub fn len(&self) -> usize {
-        self.entries.len()
-    }
-
-    /// Check if the cache is empty.
-    pub fn is_empty(&self) -> bool {
-        self.entries.is_empty()
-    }
-
-    /// Clear the cache.
-    pub fn clear(&mut self) {
-        self.entries.clear();
-        self.kv_heap_size.set(0);
-    }
-
-    /// Insert an entry into the cache.
-    pub fn insert(&mut self, key: K, value: V) -> Option<V> {
-        let key_size = self.kv_heap_size.add_val(&key);
-        self.kv_heap_size.add_val(&value);
-        let old_val = self.entries.insert(key, value);
-        if let Some(old_val) = &old_val {
-            self.kv_heap_size.sub_size(key_size);
-            self.kv_heap_size.sub_val(old_val);
-        }
-        // evict if capacity is reached
-        while self.entries.len() > self.capacity {
-            if let Some((key, val)) = self.entries.pop_last() {
-                self.kv_heap_size.sub(&key, &val);
-            }
-        }
-        old_val
-    }
-
-    /// Remove an entry from the cache.
-    pub fn remove(&mut self, key: &K) -> Option<V> {
-        let old_val = self.entries.remove(key);
-        if let Some(val) = &old_val {
-            self.kv_heap_size.sub(key, val);
-        }
-        old_val
-    }
-
-    /// Get the first (smallest) key-value pair in the cache.
-    pub fn first_key_value(&self) -> Option<(&K, &V)> {
-        self.entries.first_key_value()
-    }
-
-    /// Get the first (smallest) key in the cache.
-    pub fn first_key(&self) -> Option<&K> {
-        self.first_key_value().map(|(k, _)| k)
-    }
-
-    /// Get the last (largest) key-value pair in the cache.
-    pub fn last_key_value(&self) -> Option<(&K, &V)> {
-        self.entries.last_key_value()
-    }
-
-    /// Get the last (largest) key in the cache.
-    pub fn last_key(&self) -> Option<&K> {
-        self.last_key_value().map(|(k, _)| k)
-    }
-
-    /// Iterate over the values in the cache.
-    pub fn values(&self) -> impl Iterator<Item = &V> {
-        self.entries.values()
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use itertools::Itertools;
-
-    use super::*;
-
-    #[test]
-    fn test_top_n_cache() {
-        let mut cache = TopNCache::new(3);
-        assert_eq!(cache.capacity(), 3);
-        assert_eq!(cache.len(), 0);
-        assert!(cache.is_empty());
-        assert!(cache.first_key_value().is_none());
-        assert!(cache.first_key().is_none());
-        assert!(cache.last_key_value().is_none());
-        assert!(cache.last_key().is_none());
-        assert!(cache.values().collect_vec().is_empty());
-
-        let old_val = cache.insert(5, "hello".to_string());
-        assert!(old_val.is_none());
-        assert_eq!(cache.len(), 1);
-        assert!(!cache.is_empty());
-        assert_eq!(cache.values().collect_vec(), vec!["hello"]);
-
-        cache.insert(3, "world".to_string());
-        cache.insert(1, "risingwave!".to_string());
-        assert_eq!(cache.len(), 3);
-        assert_eq!(
-            cache.first_key_value(),
-            Some((&1, &"risingwave!".to_string()))
-        );
-        assert_eq!(cache.first_key(), Some(&1));
-        assert_eq!(cache.last_key_value(), Some((&5, &"hello".to_string())));
-        assert_eq!(cache.last_key(), Some(&5));
-        assert_eq!(
-            cache.values().collect_vec(),
-            vec!["risingwave!", "world", "hello"]
-        );
-
-        cache.insert(0, "foo".to_string());
-        assert_eq!(cache.capacity(), 3);
-        assert_eq!(cache.len(), 3);
-        assert_eq!(cache.first_key(), Some(&0));
-        assert_eq!(cache.last_key(), Some(&3));
-        assert_eq!(
-            cache.values().collect_vec(),
-            vec!["foo", "risingwave!", "world"]
-        );
-
-        let old_val = cache.remove(&0);
-        assert_eq!(old_val, Some("foo".to_string()));
-        assert_eq!(cache.len(), 2);
-        assert_eq!(cache.first_key(), Some(&1));
-        assert_eq!(cache.last_key(), Some(&3));
-        cache.remove(&3);
-        assert_eq!(cache.len(), 1);
-        assert_eq!(cache.first_key(), Some(&1));
-        assert_eq!(cache.last_key(), Some(&1));
-        let old_val = cache.remove(&100); // can remove non-existing key
-        assert!(old_val.is_none());
-        assert_eq!(cache.len(), 1);
-
-        cache.clear();
-        assert_eq!(cache.len(), 0);
-        assert_eq!(cache.capacity(), 3);
-        assert_eq!(cache.first_key(), None);
-        assert_eq!(cache.last_key(), None);
-    }
-}
diff --git a/src/stream/src/common/log_store_impl/in_mem.rs b/src/stream/src/common/log_store_impl/in_mem.rs
index d1ce2e6e5aa35..16de320aa7edd 100644
--- a/src/stream/src/common/log_store_impl/in_mem.rs
+++ b/src/stream/src/common/log_store_impl/in_mem.rs
@@ -17,7 +17,7 @@ use std::sync::Arc;
 use anyhow::{anyhow, Context};
 use await_tree::InstrumentAwait;
 use risingwave_common::array::StreamChunk;
-use risingwave_common::buffer::Bitmap;
+use risingwave_common::bitmap::Bitmap;
 use risingwave_common::util::epoch::{EpochExt, EpochPair, INVALID_EPOCH};
 use risingwave_connector::sink::log_store::{
     LogReader, LogStoreFactory, LogStoreReadItem, LogStoreResult, LogWriter, TruncateOffset,
diff --git a/src/stream/src/common/log_store_impl/kv_log_store/buffer.rs b/src/stream/src/common/log_store_impl/kv_log_store/buffer.rs
index 85926a82373da..df0c471f1da68 100644
--- a/src/stream/src/common/log_store_impl/kv_log_store/buffer.rs
+++ b/src/stream/src/common/log_store_impl/kv_log_store/buffer.rs
@@ -19,7 +19,7 @@ use std::sync::Arc;
 use await_tree::InstrumentAwait;
 use parking_lot::{Mutex, MutexGuard};
 use risingwave_common::array::StreamChunk;
-use risingwave_common::buffer::Bitmap;
+use risingwave_common::bitmap::Bitmap;
 use risingwave_connector::sink::log_store::{ChunkId, LogStoreResult, TruncateOffset};
 use tokio::sync::{oneshot, Notify};
 
diff --git a/src/stream/src/common/log_store_impl/kv_log_store/mod.rs b/src/stream/src/common/log_store_impl/kv_log_store/mod.rs
index 73dca8b1956bd..b50b90b0ebef1 100644
--- a/src/stream/src/common/log_store_impl/kv_log_store/mod.rs
+++ b/src/stream/src/common/log_store_impl/kv_log_store/mod.rs
@@ -14,7 +14,7 @@
 
 use std::sync::Arc;
 
-use risingwave_common::buffer::Bitmap;
+use risingwave_common::bitmap::Bitmap;
 use risingwave_common::catalog::{TableId, TableOption};
 use risingwave_common::metrics::{
     LabelGuardedHistogram, LabelGuardedIntCounter, LabelGuardedIntGauge,
@@ -56,8 +56,8 @@ pub(crate) type ReaderTruncationOffsetType = (u64, Option<SeqIdType>);
 
 #[derive(Clone)]
 pub(crate) struct KvLogStoreReadMetrics {
-    pub storage_read_count: LabelGuardedIntCounter<4>,
-    pub storage_read_size: LabelGuardedIntCounter<4>,
+    pub storage_read_count: LabelGuardedIntCounter<5>,
+    pub storage_read_size: LabelGuardedIntCounter<5>,
 }
 
 impl KvLogStoreReadMetrics {
@@ -72,14 +72,14 @@ impl KvLogStoreReadMetrics {
 
 #[derive(Clone)]
 pub(crate) struct KvLogStoreMetrics {
-    pub storage_write_count: LabelGuardedIntCounter<3>,
-    pub storage_write_size: LabelGuardedIntCounter<3>,
-    pub rewind_count: LabelGuardedIntCounter<3>,
-    pub rewind_delay: LabelGuardedHistogram<3>,
-    pub buffer_unconsumed_item_count: LabelGuardedIntGauge<3>,
-    pub buffer_unconsumed_row_count: LabelGuardedIntGauge<3>,
-    pub buffer_unconsumed_epoch_count: LabelGuardedIntGauge<3>,
-    pub buffer_unconsumed_min_epoch: LabelGuardedIntGauge<3>,
+    pub storage_write_count: LabelGuardedIntCounter<4>,
+    pub storage_write_size: LabelGuardedIntCounter<4>,
+    pub rewind_count: LabelGuardedIntCounter<4>,
+    pub rewind_delay: LabelGuardedHistogram<4>,
+    pub buffer_unconsumed_item_count: LabelGuardedIntGauge<4>,
+    pub buffer_unconsumed_row_count: LabelGuardedIntGauge<4>,
+    pub buffer_unconsumed_epoch_count: LabelGuardedIntGauge<4>,
+    pub buffer_unconsumed_min_epoch: LabelGuardedIntGauge<4>,
     pub persistent_log_read_metrics: KvLogStoreReadMetrics,
     pub flushed_buffer_read_metrics: KvLogStoreReadMetrics,
 }
@@ -87,13 +87,19 @@ pub(crate) struct KvLogStoreMetrics {
 impl KvLogStoreMetrics {
     pub(crate) fn new(
         metrics: &StreamingMetrics,
-        identity: &String,
+        actor_id: ActorId,
         sink_param: &SinkParam,
         connector: &'static str,
     ) -> Self {
-        let executor_id = identity;
-        let sink_id = format!("{}", sink_param.sink_id.sink_id);
-        let labels = &[executor_id.as_str(), connector, sink_id.as_str()];
+        let actor_id_str = actor_id.to_string();
+        let sink_id_str = sink_param.sink_id.sink_id.to_string();
+
+        let labels = &[
+            &actor_id_str,
+            connector,
+            &sink_id_str,
+            &sink_param.sink_name,
+        ];
         let storage_write_size = metrics
             .kv_log_store_storage_write_size
             .with_guarded_label_values(labels);
@@ -107,34 +113,38 @@ impl KvLogStoreMetrics {
         let persistent_log_read_size = metrics
             .kv_log_store_storage_read_size
             .with_guarded_label_values(&[
-                executor_id.as_str(),
+                &actor_id_str,
                 connector,
-                sink_id.as_str(),
+                &sink_id_str,
+                &sink_param.sink_name,
                 READ_PERSISTENT_LOG,
             ]);
         let persistent_log_read_count = metrics
             .kv_log_store_storage_read_count
             .with_guarded_label_values(&[
-                executor_id.as_str(),
+                &actor_id_str,
                 connector,
-                sink_id.as_str(),
+                &sink_id_str,
+                &sink_param.sink_name,
                 READ_PERSISTENT_LOG,
             ]);
 
         let flushed_buffer_read_size = metrics
             .kv_log_store_storage_read_size
             .with_guarded_label_values(&[
-                executor_id.as_str(),
+                &actor_id_str,
                 connector,
-                sink_id.as_str(),
+                &sink_id_str,
+                &sink_param.sink_name,
                 READ_FLUSHED_BUFFER,
             ]);
         let flushed_buffer_read_count = metrics
             .kv_log_store_storage_read_count
             .with_guarded_label_values(&[
-                executor_id.as_str(),
+                &actor_id_str,
                 connector,
-                sink_id.as_str(),
+                &sink_id_str,
+                &sink_param.sink_name,
                 READ_FLUSHED_BUFFER,
             ]);
 
@@ -293,6 +303,8 @@ mod v1 {
 
 pub(crate) use v2::KV_LOG_STORE_V2_INFO;
 
+use crate::task::ActorId;
+
 /// A new version of log store schema. Compared to v1, the v2 added a new vnode column to the log store pk,
 /// becomes `epoch`, `seq_id` and `vnode`. In this way, providing a log store pk, we can get exactly one single row.
 ///
@@ -424,6 +436,7 @@ impl<S: StateStore> LogStoreFactory for KvLogStoreFactory<S> {
 
 #[cfg(test)]
 mod tests {
+    use std::collections::HashSet;
     use std::future::{poll_fn, Future};
     use std::iter::empty;
     use std::pin::pin;
@@ -432,7 +445,8 @@ mod tests {
 
     use itertools::Itertools;
     use risingwave_common::array::StreamChunk;
-    use risingwave_common::buffer::{Bitmap, BitmapBuilder};
+    use risingwave_common::bitmap::{Bitmap, BitmapBuilder};
+    use risingwave_common::catalog::TableId;
     use risingwave_common::hash::VirtualNode;
     use risingwave_common::util::epoch::{EpochExt, EpochPair};
     use risingwave_connector::sink::log_store::{
@@ -495,12 +509,18 @@ mod tests {
             .version()
             .max_committed_epoch
             .next_epoch();
+        test_env
+            .storage
+            .start_epoch(epoch1, HashSet::from_iter([TableId::new(table.id)]));
         writer
             .init(EpochPair::new_test_epoch(epoch1), false)
             .await
             .unwrap();
         writer.write_chunk(stream_chunk1.clone()).await.unwrap();
         let epoch2 = epoch1.next_epoch();
+        test_env
+            .storage
+            .start_epoch(epoch2, HashSet::from_iter([TableId::new(table.id)]));
         writer.flush_current_epoch(epoch2, false).await.unwrap();
         writer.write_chunk(stream_chunk2.clone()).await.unwrap();
         let epoch3 = epoch2.next_epoch();
@@ -596,12 +616,18 @@ mod tests {
             .version()
             .max_committed_epoch
             .next_epoch();
+        test_env
+            .storage
+            .start_epoch(epoch1, HashSet::from_iter([TableId::new(table.id)]));
         writer
             .init(EpochPair::new_test_epoch(epoch1), false)
             .await
             .unwrap();
         writer.write_chunk(stream_chunk1.clone()).await.unwrap();
         let epoch2 = epoch1.next_epoch();
+        test_env
+            .storage
+            .start_epoch(epoch2, HashSet::from_iter([TableId::new(table.id)]));
         writer.flush_current_epoch(epoch2, false).await.unwrap();
         writer.write_chunk(stream_chunk2.clone()).await.unwrap();
         let epoch3 = epoch2.next_epoch();
@@ -678,6 +704,9 @@ mod tests {
             pk_info,
         );
         let (mut reader, mut writer) = factory.build().await;
+        test_env
+            .storage
+            .start_epoch(epoch3, HashSet::from_iter([TableId::new(table.id)]));
         writer
             .init(EpochPair::new_test_epoch(epoch3), false)
             .await
@@ -776,6 +805,9 @@ mod tests {
             .version()
             .max_committed_epoch
             .next_epoch();
+        test_env
+            .storage
+            .start_epoch(epoch1, HashSet::from_iter([TableId::new(table.id)]));
         writer
             .init(EpochPair::new_test_epoch(epoch1), false)
             .await
@@ -783,6 +815,9 @@ mod tests {
         writer.write_chunk(stream_chunk1_1.clone()).await.unwrap();
         writer.write_chunk(stream_chunk1_2.clone()).await.unwrap();
         let epoch2 = epoch1.next_epoch();
+        test_env
+            .storage
+            .start_epoch(epoch2, HashSet::from_iter([TableId::new(table.id)]));
         writer.flush_current_epoch(epoch2, true).await.unwrap();
         writer.write_chunk(stream_chunk2.clone()).await.unwrap();
 
@@ -883,6 +918,9 @@ mod tests {
         );
         let (mut reader, mut writer) = factory.build().await;
 
+        test_env
+            .storage
+            .start_epoch(epoch3, HashSet::from_iter([TableId::new(table.id)]));
         writer
             .init(EpochPair::new_test_epoch(epoch3), false)
             .await
@@ -993,6 +1031,9 @@ mod tests {
             .version()
             .max_committed_epoch
             .next_epoch();
+        test_env
+            .storage
+            .start_epoch(epoch1, HashSet::from_iter([TableId::new(table.id)]));
         writer1
             .init(EpochPair::new_test_epoch(epoch1), false)
             .await
@@ -1007,6 +1048,9 @@ mod tests {
         writer1.write_chunk(chunk1_1.clone()).await.unwrap();
         writer2.write_chunk(chunk1_2.clone()).await.unwrap();
         let epoch2 = epoch1.next_epoch();
+        test_env
+            .storage
+            .start_epoch(epoch2, HashSet::from_iter([TableId::new(table.id)]));
         writer1.flush_current_epoch(epoch2, false).await.unwrap();
         writer2.flush_current_epoch(epoch2, false).await.unwrap();
         let [chunk2_1, chunk2_2] = gen_multi_vnode_stream_chunks::<2>(200, 100, pk_info);
@@ -1108,6 +1152,9 @@ mod tests {
             pk_info,
         );
         let (mut reader, mut writer) = factory.build().await;
+        test_env
+            .storage
+            .start_epoch(epoch3, HashSet::from_iter([TableId::new(table.id)]));
         writer
             .init(EpochPair::new(epoch3, epoch2), false)
             .await
@@ -1177,6 +1224,9 @@ mod tests {
             .version()
             .max_committed_epoch
             .next_epoch();
+        test_env
+            .storage
+            .start_epoch(epoch1, HashSet::from_iter([TableId::new(table.id)]));
         writer
             .init(EpochPair::new_test_epoch(epoch1), false)
             .await
@@ -1314,15 +1364,24 @@ mod tests {
             .version()
             .max_committed_epoch
             .next_epoch();
+        test_env
+            .storage
+            .start_epoch(epoch1, HashSet::from_iter([TableId::new(table.id)]));
         writer
             .init(EpochPair::new_test_epoch(epoch1), false)
             .await
             .unwrap();
         writer.write_chunk(stream_chunk1.clone()).await.unwrap();
         let epoch2 = epoch1.next_epoch();
+        test_env
+            .storage
+            .start_epoch(epoch2, HashSet::from_iter([TableId::new(table.id)]));
         writer.flush_current_epoch(epoch2, true).await.unwrap();
         writer.write_chunk(stream_chunk2.clone()).await.unwrap();
         let epoch3 = epoch2.next_epoch();
+        test_env
+            .storage
+            .start_epoch(epoch3, HashSet::from_iter([TableId::new(table.id)]));
         writer.flush_current_epoch(epoch3, true).await.unwrap();
         writer.write_chunk(stream_chunk3.clone()).await.unwrap();
         writer.flush_current_epoch(u64::MAX, true).await.unwrap();
@@ -1411,6 +1470,9 @@ mod tests {
         let (mut reader, mut writer) = factory.build().await;
 
         let epoch4 = epoch3.next_epoch();
+        test_env
+            .storage
+            .start_epoch(epoch4, HashSet::from_iter([TableId::new(table.id)]));
         writer
             .init(EpochPair::new(epoch4, epoch3), false)
             .await
@@ -1471,6 +1533,9 @@ mod tests {
             .unwrap();
         writer.write_chunk(stream_chunk4.clone()).await.unwrap();
         let epoch5 = epoch4 + 1;
+        test_env
+            .storage
+            .start_epoch(epoch5, HashSet::from_iter([TableId::new(table.id)]));
         writer.flush_current_epoch(epoch5, true).await.unwrap();
         writer.write_chunk(stream_chunk5.clone()).await.unwrap();
 
@@ -1629,12 +1694,18 @@ mod tests {
             .version()
             .max_committed_epoch
             .next_epoch();
+        test_env
+            .storage
+            .start_epoch(epoch1, HashSet::from_iter([TableId::new(table.id)]));
         writer
             .init(EpochPair::new_test_epoch(epoch1), false)
             .await
             .unwrap();
         writer.write_chunk(stream_chunk1.clone()).await.unwrap();
         let epoch2 = epoch1.next_epoch();
+        test_env
+            .storage
+            .start_epoch(epoch2, HashSet::from_iter([TableId::new(table.id)]));
         writer.flush_current_epoch(epoch2, false).await.unwrap();
         writer.write_chunk(stream_chunk2.clone()).await.unwrap();
         let epoch3 = epoch2.next_epoch();
@@ -1693,6 +1764,9 @@ mod tests {
             pk_info,
         );
         let (mut reader, mut writer) = factory.build().await;
+        test_env
+            .storage
+            .start_epoch(epoch3, HashSet::from_iter([TableId::new(table.id)]));
         writer
             .init(EpochPair::new_test_epoch(epoch3), false)
             .await
@@ -1754,6 +1828,9 @@ mod tests {
             pk_info,
         );
         let (mut reader, mut writer) = factory.build().await;
+        test_env
+            .storage
+            .start_epoch(epoch4, HashSet::from_iter([TableId::new(table.id)]));
         writer
             .init(EpochPair::new_test_epoch(epoch4), false)
             .await
diff --git a/src/stream/src/common/log_store_impl/kv_log_store/reader.rs b/src/stream/src/common/log_store_impl/kv_log_store/reader.rs
index 4df114ab3df11..63cc1bd4abf0c 100644
--- a/src/stream/src/common/log_store_impl/kv_log_store/reader.rs
+++ b/src/stream/src/common/log_store_impl/kv_log_store/reader.rs
@@ -25,7 +25,7 @@ use foyer::CacheContext;
 use futures::future::{try_join_all, BoxFuture};
 use futures::{FutureExt, TryFutureExt};
 use risingwave_common::array::StreamChunk;
-use risingwave_common::buffer::Bitmap;
+use risingwave_common::bitmap::Bitmap;
 use risingwave_common::catalog::TableId;
 use risingwave_common::hash::VnodeBitmapExt;
 use risingwave_common::metrics::{LabelGuardedHistogram, LabelGuardedIntCounter};
@@ -68,8 +68,8 @@ fn initial_rewind_backoff_policy() -> RewindBackoffPolicy {
 struct RewindDelay {
     last_rewind_truncate_offset: Option<TruncateOffset>,
     backoff_policy: RewindBackoffPolicy,
-    rewind_count: LabelGuardedIntCounter<3>,
-    rewind_delay: LabelGuardedHistogram<3>,
+    rewind_count: LabelGuardedIntCounter<4>,
+    rewind_delay: LabelGuardedHistogram<4>,
 }
 
 impl RewindDelay {
diff --git a/src/stream/src/common/log_store_impl/kv_log_store/serde.rs b/src/stream/src/common/log_store_impl/kv_log_store/serde.rs
index f016257b6c008..92a3caf4cd2e3 100644
--- a/src/stream/src/common/log_store_impl/kv_log_store/serde.rs
+++ b/src/stream/src/common/log_store_impl/kv_log_store/serde.rs
@@ -23,7 +23,7 @@ use futures::{pin_mut, Stream, StreamExt, TryStreamExt};
 use futures_async_stream::try_stream;
 use itertools::Itertools;
 use risingwave_common::array::{Op, StreamChunk};
-use risingwave_common::buffer::Bitmap;
+use risingwave_common::bitmap::Bitmap;
 use risingwave_common::catalog::ColumnDesc;
 use risingwave_common::hash::VirtualNode;
 use risingwave_common::row::{OwnedRow, Row, RowExt};
@@ -835,7 +835,7 @@ impl<S: StateStoreReadIter> LogStoreRowOpStream<S> {
             }
             if *stream_epoch < epoch {
                 return Err(anyhow!(
-                    "current epoch {} has exceed epoch {} of stream not started",
+                    "current epoch {} has exceeded the epoch {} of the stream that has not started",
                     epoch,
                     stream_epoch
                 ));
@@ -935,7 +935,7 @@ mod tests {
     use rand::prelude::SliceRandom;
     use rand::thread_rng;
     use risingwave_common::array::{Op, StreamChunk};
-    use risingwave_common::buffer::Bitmap;
+    use risingwave_common::bitmap::Bitmap;
     use risingwave_common::hash::VirtualNode;
     use risingwave_common::row::{OwnedRow, Row};
     use risingwave_common::types::DataType;
diff --git a/src/stream/src/common/log_store_impl/kv_log_store/test_utils.rs b/src/stream/src/common/log_store_impl/kv_log_store/test_utils.rs
index ef7c39c580982..5fc10cd0cc58a 100644
--- a/src/stream/src/common/log_store_impl/kv_log_store/test_utils.rs
+++ b/src/stream/src/common/log_store_impl/kv_log_store/test_utils.rs
@@ -15,7 +15,7 @@
 use itertools::{zip_eq, Itertools};
 use rand::RngCore;
 use risingwave_common::array::{Op, RowRef, StreamChunk};
-use risingwave_common::buffer::{Bitmap, BitmapBuilder};
+use risingwave_common::bitmap::{Bitmap, BitmapBuilder};
 use risingwave_common::catalog::{ColumnDesc, ColumnId, TableId};
 use risingwave_common::hash::VirtualNode;
 use risingwave_common::row::{OwnedRow, Row};
diff --git a/src/stream/src/common/log_store_impl/kv_log_store/writer.rs b/src/stream/src/common/log_store_impl/kv_log_store/writer.rs
index 96702cb5bf7e3..120a49e8e7ee3 100644
--- a/src/stream/src/common/log_store_impl/kv_log_store/writer.rs
+++ b/src/stream/src/common/log_store_impl/kv_log_store/writer.rs
@@ -17,7 +17,7 @@ use std::sync::Arc;
 use anyhow::anyhow;
 use itertools::Itertools;
 use risingwave_common::array::StreamChunk;
-use risingwave_common::buffer::{Bitmap, BitmapBuilder};
+use risingwave_common::bitmap::{Bitmap, BitmapBuilder};
 use risingwave_common::catalog::TableId;
 use risingwave_common::hash::{VirtualNode, VnodeBitmapExt};
 use risingwave_common::util::epoch::EpochPair;
diff --git a/src/stream/src/common/mod.rs b/src/stream/src/common/mod.rs
index 9d79cecd2ec53..207201c4ff348 100644
--- a/src/stream/src/common/mod.rs
+++ b/src/stream/src/common/mod.rs
@@ -14,10 +14,10 @@
 
 pub use column_mapping::*;
 
-pub mod cache;
 mod column_mapping;
 pub mod compact_chunk;
 pub mod log_store_impl;
 pub mod metrics;
 pub mod rate_limit;
+pub mod state_cache;
 pub mod table;
diff --git a/src/stream/src/common/cache/state_cache/mod.rs b/src/stream/src/common/state_cache/mod.rs
similarity index 99%
rename from src/stream/src/common/cache/state_cache/mod.rs
rename to src/stream/src/common/state_cache/mod.rs
index ae3b1ad3069e4..7d547c26b0d88 100644
--- a/src/stream/src/common/cache/state_cache/mod.rs
+++ b/src/stream/src/common/state_cache/mod.rs
@@ -12,14 +12,15 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-pub use ordered::*;
 use risingwave_common::array::Op;
 use risingwave_common_estimate_size::EstimateSize;
-pub use top_n::*;
 
 mod ordered;
 mod top_n;
 
+pub use ordered::*;
+pub use top_n::*;
+
 /// A common interface for state table cache.
 pub trait StateCache: EstimateSize {
     type Key: Ord + EstimateSize;
diff --git a/src/stream/src/common/cache/state_cache/ordered.rs b/src/stream/src/common/state_cache/ordered.rs
similarity index 69%
rename from src/stream/src/common/cache/state_cache/ordered.rs
rename to src/stream/src/common/state_cache/ordered.rs
index 535dbcb74e9a0..8a47d1b7ecafe 100644
--- a/src/stream/src/common/cache/state_cache/ordered.rs
+++ b/src/stream/src/common/state_cache/ordered.rs
@@ -12,55 +12,25 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::BTreeMap;
-
 use risingwave_common::array::Op;
-use risingwave_common_estimate_size::{EstimateSize, KvSize};
+use risingwave_common_estimate_size::collections::EstimatedBTreeMap;
+use risingwave_common_estimate_size::EstimateSize;
 
 use super::{StateCache, StateCacheFiller};
 
-/// An implementation of [`StateCache`] that uses a [`BTreeMap`] as the underlying cache, with no
-/// capacity limit.
+/// An implementation of [`StateCache`] that keeps all entries in an ordered in-memory map.
+#[derive(Clone, EstimateSize)]
 pub struct OrderedStateCache<K: Ord + EstimateSize, V: EstimateSize> {
-    cache: BTreeMap<K, V>,
+    cache: EstimatedBTreeMap<K, V>,
     synced: bool,
-    kv_heap_size: KvSize,
-}
-
-impl<K: Ord + EstimateSize, V: EstimateSize> EstimateSize for OrderedStateCache<K, V> {
-    fn estimated_heap_size(&self) -> usize {
-        // TODO: Add btreemap internal size
-        // https://github.com/risingwavelabs/risingwave/issues/9713
-        self.kv_heap_size.size()
-    }
 }
 
 impl<K: Ord + EstimateSize, V: EstimateSize> OrderedStateCache<K, V> {
     pub fn new() -> Self {
         Self {
-            cache: BTreeMap::new(),
+            cache: Default::default(),
             synced: false,
-            kv_heap_size: KvSize::new(),
-        }
-    }
-
-    fn insert_cache(&mut self, key: K, value: V) -> Option<V> {
-        let key_size = self.kv_heap_size.add_val(&key);
-        self.kv_heap_size.add_val(&value);
-        let old_val = self.cache.insert(key, value);
-        if let Some(old_val) = &old_val {
-            self.kv_heap_size.sub_size(key_size);
-            self.kv_heap_size.sub_val(old_val);
-        }
-        old_val
-    }
-
-    fn delete_cache(&mut self, key: &K) -> Option<V> {
-        let old_val = self.cache.remove(key);
-        if let Some(old_val) = &old_val {
-            self.kv_heap_size.sub(key, old_val);
         }
-        old_val
     }
 }
 
@@ -87,7 +57,7 @@ impl<K: Ord + EstimateSize, V: EstimateSize> StateCache for OrderedStateCache<K,
 
     fn insert(&mut self, key: Self::Key, value: Self::Value) -> Option<Self::Value> {
         if self.synced {
-            self.insert_cache(key, value)
+            self.cache.insert(key, value)
         } else {
             None
         }
@@ -95,7 +65,7 @@ impl<K: Ord + EstimateSize, V: EstimateSize> StateCache for OrderedStateCache<K,
 
     fn delete(&mut self, key: &Self::Key) -> Option<Self::Value> {
         if self.synced {
-            self.delete_cache(key)
+            self.cache.remove(key)
         } else {
             None
         }
diff --git a/src/stream/src/common/cache/state_cache/top_n.rs b/src/stream/src/common/state_cache/top_n.rs
similarity index 87%
rename from src/stream/src/common/cache/state_cache/top_n.rs
rename to src/stream/src/common/state_cache/top_n.rs
index 4d2423f6c1e10..fef7017d75129 100644
--- a/src/stream/src/common/cache/state_cache/top_n.rs
+++ b/src/stream/src/common/state_cache/top_n.rs
@@ -13,17 +13,17 @@
 // limitations under the License.
 
 use risingwave_common::array::Op;
+use risingwave_common_estimate_size::collections::EstimatedBTreeMap;
 use risingwave_common_estimate_size::EstimateSize;
 
 use super::{StateCache, StateCacheFiller};
-use crate::common::cache::TopNCache;
 
-/// An implementation of [`StateCache`] that uses a [`TopNCache`] as the underlying cache, with
-/// limited capacity.
+/// An implementation of [`StateCache`] that keeps a limited number of entries in an ordered in-memory map.
 #[derive(Clone, EstimateSize)]
 pub struct TopNStateCache<K: Ord + EstimateSize, V: EstimateSize> {
     table_row_count: Option<usize>,
-    cache: TopNCache<K, V>,
+    cache: EstimatedBTreeMap<K, V>,
+    capacity: usize,
     synced: bool,
 }
 
@@ -31,7 +31,8 @@ impl<K: Ord + EstimateSize, V: EstimateSize> TopNStateCache<K, V> {
     pub fn new(capacity: usize) -> Self {
         Self {
             table_row_count: None,
-            cache: TopNCache::new(capacity),
+            cache: Default::default(),
+            capacity,
             synced: false,
         }
     }
@@ -39,7 +40,8 @@ impl<K: Ord + EstimateSize, V: EstimateSize> TopNStateCache<K, V> {
     pub fn with_table_row_count(capacity: usize, table_row_count: usize) -> Self {
         Self {
             table_row_count: Some(table_row_count),
-            cache: TopNCache::new(capacity),
+            cache: Default::default(),
+            capacity,
             synced: false,
         }
     }
@@ -65,7 +67,12 @@ impl<K: Ord + EstimateSize, V: EstimateSize> TopNStateCache<K, V> {
             || self.cache.is_empty()
             || &key <= self.cache.last_key().unwrap()
         {
-            self.cache.insert(key, value)
+            let old_v = self.cache.insert(key, value);
+            // evict if capacity is reached
+            while self.cache.len() > self.capacity {
+                self.cache.pop_last();
+            }
+            old_v
         } else {
             None
         };
@@ -87,8 +94,8 @@ impl<K: Ord + EstimateSize, V: EstimateSize> TopNStateCache<K, V> {
         old_val
     }
 
-    pub fn capacity_inner(&self) -> usize {
-        self.cache.capacity()
+    pub fn capacity(&self) -> usize {
+        self.capacity
     }
 
     pub fn len(&self) -> usize {
@@ -170,7 +177,7 @@ impl<K: Ord + EstimateSize, V: EstimateSize> StateCacheFiller for &mut TopNState
     type Value = V;
 
     fn capacity(&self) -> Option<usize> {
-        Some(self.capacity_inner())
+        Some(TopNStateCache::capacity(self))
     }
 
     fn insert_unchecked(&mut self, key: Self::Key, value: Self::Value) {
diff --git a/src/stream/src/common/table/mod.rs b/src/stream/src/common/table/mod.rs
index 2fe40ab1cf6ad..6a130cbd255a2 100644
--- a/src/stream/src/common/table/mod.rs
+++ b/src/stream/src/common/table/mod.rs
@@ -14,7 +14,6 @@
 
 pub mod state_table;
 mod state_table_cache;
-mod watermark;
 
 #[cfg(test)]
 pub mod test_state_table;
diff --git a/src/stream/src/common/table/state_table.rs b/src/stream/src/common/table/state_table.rs
index 98c8517cfbbc3..df85aa7f260ce 100644
--- a/src/stream/src/common/table/state_table.rs
+++ b/src/stream/src/common/table/state_table.rs
@@ -25,7 +25,7 @@ use futures_async_stream::for_await;
 use itertools::{izip, Itertools};
 use risingwave_common::array::stream_record::Record;
 use risingwave_common::array::{ArrayImplBuilder, ArrayRef, DataChunk, Op, StreamChunk};
-use risingwave_common::buffer::Bitmap;
+use risingwave_common::bitmap::Bitmap;
 use risingwave_common::catalog::{
     get_dist_key_in_pk_indices, ColumnDesc, ColumnId, TableId, TableOption,
 };
@@ -39,8 +39,8 @@ use risingwave_common::util::row_serde::OrderedRowSerde;
 use risingwave_common::util::sort_util::OrderType;
 use risingwave_common::util::value_encoding::BasicSerde;
 use risingwave_hummock_sdk::key::{
-    end_bound_of_prefix, prefixed_range_with_vnode, range_of_prefix,
-    start_bound_of_excluded_prefix, TableKey, TableKeyRange,
+    end_bound_of_prefix, prefixed_range_with_vnode, start_bound_of_excluded_prefix, TableKey,
+    TableKeyRange,
 };
 use risingwave_hummock_sdk::table_watermark::{VnodeWatermark, WatermarkDirection};
 use risingwave_pb::catalog::Table;
@@ -57,26 +57,23 @@ use risingwave_storage::store::{
     ReadLogOptions, ReadOptions, SealCurrentEpochOptions, StateStoreIter, StateStoreIterExt,
 };
 use risingwave_storage::table::merge_sort::merge_sort;
-use risingwave_storage::table::{deserialize_log_stream, KeyedRow, TableDistribution};
+use risingwave_storage::table::{
+    deserialize_log_stream, ChangeLogRow, KeyedRow, TableDistribution,
+};
 use risingwave_storage::StateStore;
 use thiserror_ext::AsReport;
 use tracing::{trace, Instrument};
 
-use super::watermark::{WatermarkBufferByEpoch, WatermarkBufferStrategy};
 use crate::cache::cache_may_stale;
-use crate::common::cache::{StateCache, StateCacheFiller};
+use crate::common::state_cache::{StateCache, StateCacheFiller};
 use crate::common::table::state_table_cache::StateTableWatermarkCache;
 use crate::executor::{StreamExecutorError, StreamExecutorResult};
 
-/// This num is arbitrary and we may want to improve this choice in the future.
-const STATE_CLEANING_PERIOD_EPOCH: usize = 300;
 /// Mostly watermark operators will have inserts (append-only).
 /// So this number should not need to be very large.
 /// But we may want to improve this choice in the future.
 const WATERMARK_CACHE_ENTRIES: usize = 16;
 
-type DefaultWatermarkBufferStrategy = WatermarkBufferByEpoch<STATE_CLEANING_PERIOD_EPOCH>;
-
 /// This macro is used to mark a point where we want to randomly discard the operation and early
 /// return, only in insane mode.
 macro_rules! insane_mode_discard_point {
@@ -95,12 +92,10 @@ pub struct StateTableInner<
     S,
     SD = BasicSerde,
     const IS_REPLICATED: bool = false,
-    W = DefaultWatermarkBufferStrategy,
     const USE_WATERMARK_CACHE: bool = false,
 > where
     S: StateStore,
     SD: ValueRowSerde,
-    W: WatermarkBufferStrategy,
 {
     /// Id for this table.
     table_id: TableId,
@@ -136,15 +131,11 @@ pub struct StateTableInner<
 
     value_indices: Option<Vec<usize>>,
 
-    /// Strategy to buffer watermark for lazy state cleaning.
-    watermark_buffer_strategy: W,
-    /// State cleaning watermark. Old states will be cleaned under this watermark when committing.
-    state_clean_watermark: Option<ScalarImpl>,
-
-    /// Watermark of the last committed state cleaning.
-    prev_cleaned_watermark: Option<ScalarImpl>,
-
-    /// Watermark cache
+    /// Pending watermark for state cleaning. Old states below this watermark will be cleaned when committing.
+    pending_watermark: Option<ScalarImpl>,
+    /// Last committed watermark for state cleaning. Will be restored on state table recovery.
+    committed_watermark: Option<ScalarImpl>,
+    /// Cache for the top-N primary keys for reducing unnecessary range deletion.
     watermark_cache: StateTableWatermarkCache,
 
     /// Data Types
@@ -174,17 +165,15 @@ pub type StateTable<S> = StateTableInner<S, BasicSerde>;
 pub type ReplicatedStateTable<S, SD> = StateTableInner<S, SD, true>;
 /// `WatermarkCacheStateTable` caches the watermark column.
 /// It will reduce state cleaning overhead.
-pub type WatermarkCacheStateTable<S> =
-    StateTableInner<S, BasicSerde, false, DefaultWatermarkBufferStrategy, true>;
+pub type WatermarkCacheStateTable<S> = StateTableInner<S, BasicSerde, false, true>;
 pub type WatermarkCacheParameterizedStateTable<S, const USE_WATERMARK_CACHE: bool> =
-    StateTableInner<S, BasicSerde, false, DefaultWatermarkBufferStrategy, USE_WATERMARK_CACHE>;
+    StateTableInner<S, BasicSerde, false, USE_WATERMARK_CACHE>;
 
 // initialize
-impl<S, SD, W, const USE_WATERMARK_CACHE: bool> StateTableInner<S, SD, true, W, USE_WATERMARK_CACHE>
+impl<S, SD, const USE_WATERMARK_CACHE: bool> StateTableInner<S, SD, true, USE_WATERMARK_CACHE>
 where
     S: StateStore,
     SD: ValueRowSerde,
-    W: WatermarkBufferStrategy,
 {
     /// get the newest epoch of the state store and panic if the `init_epoch()` has never be called
     /// async interface only used for replicated state table,
@@ -195,12 +184,10 @@ where
 }
 
 // initialize
-impl<S, SD, W, const USE_WATERMARK_CACHE: bool>
-    StateTableInner<S, SD, false, W, USE_WATERMARK_CACHE>
+impl<S, SD, const USE_WATERMARK_CACHE: bool> StateTableInner<S, SD, false, USE_WATERMARK_CACHE>
 where
     S: StateStore,
     SD: ValueRowSerde,
-    W: WatermarkBufferStrategy,
 {
     /// get the newest epoch of the state store and panic if the `init_epoch()` has never be called
     /// No need to `wait_for_epoch`, so it should complete immediately.
@@ -268,12 +255,11 @@ pub enum StateTableOpConsistencyLevel {
 // FIXME(kwannoel): Enforce that none of the constructors here
 // should be used by replicated state table.
 // Apart from from_table_catalog_inner.
-impl<S, SD, const IS_REPLICATED: bool, W, const USE_WATERMARK_CACHE: bool>
-    StateTableInner<S, SD, IS_REPLICATED, W, USE_WATERMARK_CACHE>
+impl<S, SD, const IS_REPLICATED: bool, const USE_WATERMARK_CACHE: bool>
+    StateTableInner<S, SD, IS_REPLICATED, USE_WATERMARK_CACHE>
 where
     S: StateStore,
     SD: ValueRowSerde,
-    W: WatermarkBufferStrategy,
 {
     /// Create state table from table catalog and store.
     ///
@@ -451,6 +437,36 @@ where
             row_serde.kind().is_column_aware()
         );
 
+        // Restore persisted table watermark.
+        let prefix_deser = if pk_indices.is_empty() {
+            None
+        } else {
+            Some(pk_serde.prefix(1))
+        };
+        let max_watermark_of_vnodes = distribution
+            .vnodes()
+            .iter_vnodes()
+            .filter_map(|vnode| local_state_store.get_table_watermark(vnode))
+            .max();
+        let committed_watermark = if let Some(deser) = prefix_deser
+            && let Some(max_watermark) = max_watermark_of_vnodes
+        {
+            let deserialized = deser
+                .deserialize(&max_watermark)
+                .ok()
+                .and_then(|row| row[0].clone());
+            if deserialized.is_none() {
+                tracing::error!(
+                    vnodes = ?distribution.vnodes(),
+                    watermark = ?max_watermark,
+                    "Failed to deserialize persisted watermark from state store.",
+                );
+            }
+            deserialized
+        } else {
+            None
+        };
+
         let watermark_cache = if USE_WATERMARK_CACHE {
             StateTableWatermarkCache::new(WATERMARK_CACHE_ENTRIES)
         } else {
@@ -497,9 +513,8 @@ where
             prefix_hint_len,
             table_option,
             value_indices,
-            watermark_buffer_strategy: W::default(),
-            state_clean_watermark: None,
-            prev_cleaned_watermark: None,
+            pending_watermark: None,
+            committed_watermark,
             watermark_cache,
             data_types,
             output_indices,
@@ -664,11 +679,15 @@ where
             .collect();
         let pk_serde = OrderedRowSerde::new(pk_data_types, order_types);
 
+        // TODO: let's not restore watermark in unit tests for now, to avoid complexity.
+        let committed_watermark = None;
+
         let watermark_cache = if USE_WATERMARK_CACHE {
             StateTableWatermarkCache::new(WATERMARK_CACHE_ENTRIES)
         } else {
             StateTableWatermarkCache::new(0)
         };
+
         Self {
             table_id,
             local_store: local_state_store,
@@ -680,9 +699,8 @@ where
             prefix_hint_len: 0,
             table_option: Default::default(),
             value_indices,
-            watermark_buffer_strategy: W::default(),
-            state_clean_watermark: None,
-            prev_cleaned_watermark: None,
+            pending_watermark: None,
+            committed_watermark,
             watermark_cache,
             data_types,
             output_indices: vec![],
@@ -750,7 +768,7 @@ where
     }
 
     fn is_dirty(&self) -> bool {
-        self.local_store.is_dirty() || self.state_clean_watermark.is_some()
+        self.local_store.is_dirty() || self.pending_watermark.is_some()
     }
 
     pub fn is_consistent_op(&self) -> bool {
@@ -762,11 +780,10 @@ where
     }
 }
 
-impl<S, SD, W, const USE_WATERMARK_CACHE: bool> StateTableInner<S, SD, true, W, USE_WATERMARK_CACHE>
+impl<S, SD, const USE_WATERMARK_CACHE: bool> StateTableInner<S, SD, true, USE_WATERMARK_CACHE>
 where
     S: StateStore,
     SD: ValueRowSerde,
-    W: WatermarkBufferStrategy,
 {
     /// Create replicated state table from table catalog with output indices
     pub async fn from_table_catalog_with_output_column_ids(
@@ -787,13 +804,8 @@ where
 }
 
 // point get
-impl<
-        S,
-        SD,
-        const IS_REPLICATED: bool,
-        W: WatermarkBufferStrategy,
-        const USE_WATERMARK_CACHE: bool,
-    > StateTableInner<S, SD, IS_REPLICATED, W, USE_WATERMARK_CACHE>
+impl<S, SD, const IS_REPLICATED: bool, const USE_WATERMARK_CACHE: bool>
+    StateTableInner<S, SD, IS_REPLICATED, USE_WATERMARK_CACHE>
 where
     S: StateStore,
     SD: ValueRowSerde,
@@ -893,7 +905,7 @@ where
         let cache_may_stale = cache_may_stale(self.vnodes(), &new_vnodes);
 
         if cache_may_stale {
-            self.state_clean_watermark = None;
+            self.pending_watermark = None;
             if USE_WATERMARK_CACHE {
                 self.watermark_cache.clear();
             }
@@ -907,13 +919,8 @@ where
 }
 
 // write
-impl<
-        S,
-        SD,
-        const IS_REPLICATED: bool,
-        W: WatermarkBufferStrategy,
-        const USE_WATERMARK_CACHE: bool,
-    > StateTableInner<S, SD, IS_REPLICATED, W, USE_WATERMARK_CACHE>
+impl<S, SD, const IS_REPLICATED: bool, const USE_WATERMARK_CACHE: bool>
+    StateTableInner<S, SD, IS_REPLICATED, USE_WATERMARK_CACHE>
 where
     S: StateStore,
     SD: ValueRowSerde,
@@ -1129,12 +1136,15 @@ where
     /// # Arguments
     ///
     /// * `watermark` - Latest watermark received.
-    /// * `eager_cleaning` - Whether to clean up the state table eagerly.
-    pub fn update_watermark(&mut self, watermark: ScalarImpl, eager_cleaning: bool) {
+    pub fn update_watermark(&mut self, watermark: ScalarImpl) {
         trace!(table_id = %self.table_id, watermark = ?watermark, "update watermark");
-        if self.watermark_buffer_strategy.apply() || eager_cleaning {
-            self.state_clean_watermark = Some(watermark);
-        }
+        self.pending_watermark = Some(watermark);
+    }
+
+    /// Get the committed watermark of the state table. Watermarks should be fed into the state
+    /// table through `update_watermark` method.
+    pub fn get_committed_watermark(&self) -> Option<&ScalarImpl> {
+        self.committed_watermark.as_ref()
     }
 
     pub async fn commit(&mut self, new_epoch: EpochPair) -> StreamExecutorResult<()> {
@@ -1147,6 +1157,13 @@ where
         op_consistency_level: StateTableOpConsistencyLevel,
     ) -> StreamExecutorResult<()> {
         if self.op_consistency_level != op_consistency_level {
+            info!(
+                ?new_epoch,
+                prev_op_consistency_level = ?self.op_consistency_level,
+                ?op_consistency_level,
+                table_id = self.table_id.table_id,
+                "switch to new op consistency level"
+            );
             self.commit_inner(new_epoch, Some(op_consistency_level))
                 .await
         } else {
@@ -1178,9 +1195,7 @@ where
             epoch = ?self.epoch(),
             "commit state table"
         );
-        // Tick the watermark buffer here because state table is expected to be committed once
-        // per epoch.
-        self.watermark_buffer_strategy.tick();
+
         if !self.is_dirty() {
             // If the state table is not modified, go fast path.
             self.local_store.seal_current_epoch(
@@ -1199,7 +1214,7 @@ where
 
         // Refresh watermark cache if it is out of sync.
         if USE_WATERMARK_CACHE && !self.watermark_cache.is_synced() {
-            if let Some(ref watermark) = self.prev_cleaned_watermark {
+            if let Some(ref watermark) = self.committed_watermark {
                 let range: (Bound<Once<Datum>>, Bound<Once<Datum>>) =
                     (Included(once(Some(watermark.clone()))), Unbounded);
                 // NOTE(kwannoel): We buffer `pks` before inserting into watermark cache
@@ -1254,7 +1269,7 @@ where
         next_epoch: u64,
         switch_op_consistency_level: Option<OpConsistencyLevel>,
     ) -> StreamExecutorResult<()> {
-        let watermark = self.state_clean_watermark.take();
+        let watermark = self.pending_watermark.take();
         watermark.as_ref().inspect(|watermark| {
             trace!(table_id = %self.table_id, watermark = ?watermark, "state cleaning");
         });
@@ -1326,7 +1341,7 @@ where
                 ));
             }
         }
-        self.prev_cleaned_watermark = watermark;
+        self.committed_watermark = watermark;
 
         // Clear the watermark cache and force a resync.
         // TODO(kwannoel): This can be further optimized:
@@ -1366,13 +1381,8 @@ impl<'a, T> KeyedRowStream<'a> for T where
 }
 
 // Iterator functions
-impl<
-        S,
-        SD,
-        const IS_REPLICATED: bool,
-        W: WatermarkBufferStrategy,
-        const USE_WATERMARK_CACHE: bool,
-    > StateTableInner<S, SD, IS_REPLICATED, W, USE_WATERMARK_CACHE>
+impl<S, SD, const IS_REPLICATED: bool, const USE_WATERMARK_CACHE: bool>
+    StateTableInner<S, SD, IS_REPLICATED, USE_WATERMARK_CACHE>
 where
     S: StateStore,
     SD: ValueRowSerde,
@@ -1567,64 +1577,14 @@ where
             .map_err(StreamExecutorError::from)
     }
 
-    /// Returns:
-    /// false: the provided pk prefix is absent in state store.
-    /// true: the provided pk prefix may or may not be present in state store.
-    pub async fn may_exist(&self, pk_prefix: impl Row) -> StreamExecutorResult<bool> {
-        let prefix_serializer = self.pk_serde.prefix(pk_prefix.len());
-        let encoded_prefix = serialize_pk(&pk_prefix, &prefix_serializer);
-        let encoded_key_range = range_of_prefix(&encoded_prefix);
-
-        // We assume that all usages of iterating the state table only access a single vnode.
-        // If this assertion fails, then something must be wrong with the operator implementation or
-        // the distribution derivation from the optimizer.
-        let vnode = self.compute_prefix_vnode(&pk_prefix);
-        let table_key_range = prefixed_range_with_vnode(encoded_key_range, vnode);
-
-        // Construct prefix hint for prefix bloom filter.
-        if self.prefix_hint_len != 0 {
-            debug_assert_eq!(self.prefix_hint_len, pk_prefix.len());
-        }
-        let prefix_hint = {
-            if self.prefix_hint_len == 0 || self.prefix_hint_len > pk_prefix.len() {
-                panic!();
-            } else {
-                let encoded_prefix_len = self
-                    .pk_serde
-                    .deserialize_prefix_len(&encoded_prefix, self.prefix_hint_len)?;
-
-                Some(Bytes::copy_from_slice(
-                    &encoded_prefix[..encoded_prefix_len],
-                ))
-            }
-        };
-
-        let read_options = ReadOptions {
-            prefix_hint,
-            table_id: self.table_id,
-            cache_policy: CachePolicy::Fill(CacheContext::Default),
-            ..Default::default()
-        };
-
-        self.local_store
-            .may_exist(table_key_range, read_options)
-            .await
-            .map_err(Into::into)
-    }
-
     #[cfg(test)]
     pub fn get_watermark_cache(&self) -> &StateTableWatermarkCache {
         &self.watermark_cache
     }
 }
 
-impl<
-        S,
-        SD,
-        const IS_REPLICATED: bool,
-        W: WatermarkBufferStrategy,
-        const USE_WATERMARK_CACHE: bool,
-    > StateTableInner<S, SD, IS_REPLICATED, W, USE_WATERMARK_CACHE>
+impl<S, SD, const IS_REPLICATED: bool, const USE_WATERMARK_CACHE: bool>
+    StateTableInner<S, SD, IS_REPLICATED, USE_WATERMARK_CACHE>
 where
     S: StateStore,
     SD: ValueRowSerde,
@@ -1634,7 +1594,7 @@ where
         vnode: VirtualNode,
         epoch_range: (u64, u64),
         pk_range: &(Bound<impl Row>, Bound<impl Row>),
-    ) -> StreamExecutorResult<impl Stream<Item = StreamExecutorResult<(Op, OwnedRow)>> + '_> {
+    ) -> StreamExecutorResult<impl Stream<Item = StreamExecutorResult<ChangeLogRow>> + '_> {
         let memcomparable_range = prefix_range_to_memcomparable(&self.pk_serde, pk_range);
         let memcomparable_range_with_vnode = prefixed_range_with_vnode(memcomparable_range, vnode);
         Ok(deserialize_log_stream(
diff --git a/src/stream/src/common/table/state_table_cache.rs b/src/stream/src/common/table/state_table_cache.rs
index 585b51eae0606..86c32b404cb7e 100644
--- a/src/stream/src/common/table/state_table_cache.rs
+++ b/src/stream/src/common/table/state_table_cache.rs
@@ -17,7 +17,7 @@ use risingwave_common::row::{OwnedRow, Row, RowExt};
 use risingwave_common::types::{DefaultOrdered, ScalarRefImpl};
 use risingwave_common_estimate_size::EstimateSize;
 
-use crate::common::cache::{StateCache, TopNStateCache};
+use crate::common::state_cache::{StateCache, TopNStateCache};
 
 /// The watermark cache key is just an `OwnedRow` wrapped in `DefaultOrdered`.
 /// This is because we want to use the `DefaultOrdered` implementation of `Ord`.
@@ -125,7 +125,7 @@ impl StateTableWatermarkCache {
     }
 
     pub fn capacity(&self) -> usize {
-        self.inner.capacity_inner()
+        self.inner.capacity()
     }
 
     pub fn len(&self) -> usize {
@@ -186,7 +186,7 @@ mod tests {
     use risingwave_common::types::{Scalar, Timestamptz};
 
     use super::*;
-    use crate::common::cache::StateCacheFiller;
+    use crate::common::state_cache::StateCacheFiller;
 
     /// With capacity 3, test the following sequence of inserts:
     /// Insert
diff --git a/src/stream/src/common/table/test_state_table.rs b/src/stream/src/common/table/test_state_table.rs
index 4ef3381871cbf..098548c21ac93 100644
--- a/src/stream/src/common/table/test_state_table.rs
+++ b/src/stream/src/common/table/test_state_table.rs
@@ -12,11 +12,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::collections::HashSet;
 use std::ops::Bound::{self, *};
 
 use futures::{pin_mut, StreamExt};
 use risingwave_common::array::{Op, StreamChunk};
-use risingwave_common::buffer::Bitmap;
+use risingwave_common::bitmap::Bitmap;
 use risingwave_common::catalog::{ColumnDesc, ColumnId, TableId};
 use risingwave_common::row::{self, OwnedRow};
 use risingwave_common::types::{DataType, Scalar, Timestamptz};
@@ -24,11 +25,9 @@ use risingwave_common::util::epoch::{test_epoch, EpochPair};
 use risingwave_common::util::sort_util::OrderType;
 use risingwave_common::util::value_encoding::BasicSerde;
 use risingwave_hummock_test::test_utils::prepare_hummock_test_env;
-use risingwave_rpc_client::HummockMetaClient;
 use risingwave_storage::hummock::HummockStorage;
 use risingwave_storage::store::PrefetchOptions;
 use risingwave_storage::table::DEFAULT_VNODE;
-use risingwave_storage::StateStore;
 
 use crate::common::table::state_table::{
     ReplicatedStateTable, StateTable, WatermarkCacheStateTable,
@@ -63,6 +62,9 @@ async fn test_state_table_update_insert() {
             .await;
 
     let mut epoch = EpochPair::new_test_epoch(test_epoch(1));
+    test_env
+        .storage
+        .start_epoch(epoch.curr, HashSet::from_iter([TEST_TABLE_ID]));
     state_table.init_epoch(epoch);
 
     state_table.insert(OwnedRow::new(vec![
@@ -80,6 +82,9 @@ async fn test_state_table_update_insert() {
     ]));
 
     epoch.inc_for_test();
+    test_env
+        .storage
+        .start_epoch(epoch.curr, HashSet::from_iter([TEST_TABLE_ID]));
     state_table.commit(epoch).await.unwrap();
 
     state_table.delete(OwnedRow::new(vec![
@@ -136,6 +141,9 @@ async fn test_state_table_update_insert() {
     );
 
     epoch.inc_for_test();
+    test_env
+        .storage
+        .start_epoch(epoch.curr, HashSet::from_iter([TEST_TABLE_ID]));
     state_table.commit(epoch).await.unwrap();
 
     let row6_commit = state_table
@@ -173,6 +181,9 @@ async fn test_state_table_update_insert() {
     ]));
 
     epoch.inc_for_test();
+    test_env
+        .storage
+        .start_epoch(epoch.curr, HashSet::from_iter([TEST_TABLE_ID]));
     state_table.commit(epoch).await.unwrap();
 
     // one epoch: delete (1, 2, 3, 4), insert (5, 6, 7, None), delete(5, 6, 7, None)
@@ -202,6 +213,9 @@ async fn test_state_table_update_insert() {
     assert_eq!(row1, None);
 
     epoch.inc_for_test();
+    test_env
+        .storage
+        .start_epoch(epoch.curr, HashSet::from_iter([TEST_TABLE_ID]));
     state_table.commit(epoch).await.unwrap();
 
     let row1_commit = state_table
@@ -241,6 +255,9 @@ async fn test_state_table_iter_with_prefix() {
             .await;
 
     let mut epoch = EpochPair::new_test_epoch(test_epoch(1));
+    test_env
+        .storage
+        .start_epoch(epoch.curr, HashSet::from_iter([TEST_TABLE_ID]));
     state_table.init_epoch(epoch);
 
     state_table.insert(OwnedRow::new(vec![
@@ -267,6 +284,9 @@ async fn test_state_table_iter_with_prefix() {
     ]));
 
     epoch.inc_for_test();
+    test_env
+        .storage
+        .start_epoch(epoch.curr, HashSet::from_iter([TEST_TABLE_ID]));
     state_table.commit(epoch).await.unwrap();
 
     state_table.insert(OwnedRow::new(vec![
@@ -370,6 +390,9 @@ async fn test_state_table_iter_with_pk_range() {
             .await;
 
     let mut epoch = EpochPair::new_test_epoch(test_epoch(1));
+    test_env
+        .storage
+        .start_epoch(epoch.curr, HashSet::from_iter([TEST_TABLE_ID]));
     state_table.init_epoch(epoch);
 
     state_table.insert(OwnedRow::new(vec![
@@ -396,6 +419,9 @@ async fn test_state_table_iter_with_pk_range() {
     ]));
 
     epoch.inc_for_test();
+    test_env
+        .storage
+        .start_epoch(epoch.curr, HashSet::from_iter([TEST_TABLE_ID]));
     state_table.commit(epoch).await.unwrap();
 
     state_table.insert(OwnedRow::new(vec![
@@ -503,6 +529,9 @@ async fn test_mem_table_assertion() {
         StateTable::from_table_catalog(&table, test_env.storage.clone(), None).await;
 
     let epoch = EpochPair::new_test_epoch(test_epoch(1));
+    test_env
+        .storage
+        .start_epoch(epoch.curr, HashSet::from_iter([TEST_TABLE_ID]));
     state_table.init_epoch(epoch);
     state_table.insert(OwnedRow::new(vec![
         Some(1_i32.into()),
@@ -546,6 +575,9 @@ async fn test_state_table_iter_with_value_indices() {
             .await;
 
     let mut epoch = EpochPair::new_test_epoch(test_epoch(1));
+    test_env
+        .storage
+        .start_epoch(epoch.curr, HashSet::from_iter([TEST_TABLE_ID]));
     state_table.init_epoch(epoch);
 
     state_table.insert(OwnedRow::new(vec![
@@ -601,6 +633,9 @@ async fn test_state_table_iter_with_value_indices() {
     }
 
     epoch.inc_for_test();
+    test_env
+        .storage
+        .start_epoch(epoch.curr, HashSet::from_iter([TEST_TABLE_ID]));
     state_table.commit(epoch).await.unwrap();
 
     // write [3, 33, 333], [4, 44, 444], [5, 55, 555], [7, 77, 777], [8, 88, 888]into mem_table,
@@ -713,6 +748,9 @@ async fn test_state_table_iter_with_shuffle_value_indices() {
             .await;
 
     let mut epoch = EpochPair::new_test_epoch(test_epoch(1));
+    test_env
+        .storage
+        .start_epoch(epoch.curr, HashSet::from_iter([TEST_TABLE_ID]));
     state_table.init_epoch(epoch);
 
     state_table.insert(OwnedRow::new(vec![
@@ -789,6 +827,9 @@ async fn test_state_table_iter_with_shuffle_value_indices() {
     }
 
     epoch.inc_for_test();
+    test_env
+        .storage
+        .start_epoch(epoch.curr, HashSet::from_iter([TEST_TABLE_ID]));
     state_table.commit(epoch).await.unwrap();
 
     // write [3, 33, 333], [4, 44, 444], [5, 55, 555], [7, 77, 777], [8, 88, 888]into mem_table,
@@ -954,6 +995,9 @@ async fn test_state_table_write_chunk() {
             .await;
 
     let epoch = EpochPair::new_test_epoch(test_epoch(1));
+    test_env
+        .storage
+        .start_epoch(epoch.curr, HashSet::from_iter([TEST_TABLE_ID]));
     state_table.init_epoch(epoch);
 
     let chunk = StreamChunk::from_rows(
@@ -1083,6 +1127,9 @@ async fn test_state_table_write_chunk_visibility() {
             .await;
 
     let epoch = EpochPair::new_test_epoch(test_epoch(1));
+    test_env
+        .storage
+        .start_epoch(epoch.curr, HashSet::from_iter([TEST_TABLE_ID]));
     state_table.init_epoch(epoch);
 
     let chunk = StreamChunk::from_rows(
@@ -1207,6 +1254,9 @@ async fn test_state_table_write_chunk_value_indices() {
             .await;
 
     let epoch = EpochPair::new_test_epoch(test_epoch(1));
+    test_env
+        .storage
+        .start_epoch(epoch.curr, HashSet::from_iter([TEST_TABLE_ID]));
     state_table.init_epoch(epoch);
 
     let chunk = StreamChunk::from_rows(
@@ -1269,190 +1319,6 @@ async fn test_state_table_write_chunk_value_indices() {
     );
 }
 
-async fn check_may_exist<S>(
-    state_table: &StateTable<S>,
-    existent_prefix: Vec<i32>,
-    non_existent_prefix: Vec<i32>,
-) where
-    S: StateStore,
-{
-    for prefix in existent_prefix {
-        let pk_prefix = OwnedRow::new(vec![Some(prefix.into())]);
-        assert!(state_table.may_exist(&pk_prefix).await.unwrap());
-    }
-    for prefix in non_existent_prefix {
-        let pk_prefix = OwnedRow::new(vec![Some(prefix.into())]);
-        assert!(!state_table.may_exist(&pk_prefix).await.unwrap());
-    }
-}
-
-#[tokio::test]
-async fn test_state_table_may_exist() {
-    const TEST_TABLE_ID: TableId = TableId { table_id: 233 };
-    let test_env = prepare_hummock_test_env().await;
-
-    // let pk_columns = vec![0, 1]; leave a message to indicate pk columns
-    let order_types = vec![OrderType::ascending(), OrderType::descending()];
-
-    let column_ids = [ColumnId::from(0), ColumnId::from(1), ColumnId::from(2)];
-    let column_descs = vec![
-        ColumnDesc::unnamed(column_ids[0], DataType::Int32),
-        ColumnDesc::unnamed(column_ids[1], DataType::Int32),
-        ColumnDesc::unnamed(column_ids[2], DataType::Int32),
-    ];
-    let pk_index = vec![0_usize, 1_usize];
-    let read_prefix_len_hint = 1;
-    let table = gen_prost_table(
-        TEST_TABLE_ID,
-        column_descs,
-        order_types,
-        pk_index,
-        read_prefix_len_hint,
-    );
-
-    test_env.register_table(table.clone()).await;
-    let mut state_table =
-        StateTable::from_table_catalog_inconsistent_op(&table, test_env.storage.clone(), None)
-            .await;
-
-    let mut epoch = EpochPair::new_test_epoch(test_epoch(1));
-    state_table.init_epoch(epoch);
-
-    state_table.insert(OwnedRow::new(vec![
-        Some(1_i32.into()),
-        Some(11_i32.into()),
-        Some(111_i32.into()),
-    ]));
-    state_table.insert(OwnedRow::new(vec![
-        Some(1_i32.into()),
-        Some(22_i32.into()),
-        Some(222_i32.into()),
-    ]));
-
-    state_table.insert(OwnedRow::new(vec![
-        Some(4_i32.into()),
-        Some(44_i32.into()),
-        Some(444_i32.into()),
-    ]));
-
-    state_table.insert(OwnedRow::new(vec![
-        Some(1_i32.into()),
-        Some(55_i32.into()),
-        Some(555_i32.into()),
-    ]));
-
-    // test may_exist with data only in memtable (e1)
-    check_may_exist(&state_table, vec![1, 4], vec![2, 3, 6, 12]).await;
-
-    epoch.inc_for_test();
-    state_table.commit(epoch).await.unwrap();
-    let e1 = epoch.prev;
-
-    // test may_exist with data only in immutable memtable (e1)
-    check_may_exist(&state_table, vec![1, 4], vec![2, 3, 6, 12]).await;
-
-    let e1_res = test_env.storage.seal_and_sync_epoch(e1).await.unwrap();
-
-    // test may_exist with data only in uncommitted ssts (e1)
-    check_may_exist(&state_table, vec![1, 4], vec![2, 3, 6, 12]).await;
-
-    test_env.meta_client.commit_epoch(e1, e1_res).await.unwrap();
-    test_env.storage.try_wait_epoch_for_test(e1).await;
-
-    // test may_exist with data only in committed ssts (e1)
-    check_may_exist(&state_table, vec![1, 4], vec![2, 3, 6, 12]).await;
-
-    state_table.insert(OwnedRow::new(vec![
-        Some(1_i32.into()),
-        Some(33_i32.into()),
-        Some(333_i32.into()),
-    ]));
-    state_table.insert(OwnedRow::new(vec![
-        Some(1_i32.into()),
-        Some(55_i32.into()),
-        Some(5555_i32.into()),
-    ]));
-    state_table.insert(OwnedRow::new(vec![
-        Some(6_i32.into()),
-        Some(66_i32.into()),
-        Some(666_i32.into()),
-    ]));
-
-    // test may_exist with data in memtable (e2), committed ssts (e1)
-    check_may_exist(&state_table, vec![1, 4, 6], vec![2, 3, 12]).await;
-
-    epoch.inc_for_test();
-    state_table.commit(epoch).await.unwrap();
-    let e2 = epoch.prev;
-
-    // test may_exist with data in immutable memtable (e2), committed ssts (e1)
-    check_may_exist(&state_table, vec![1, 4, 6], vec![2, 3, 12]).await;
-
-    state_table.insert(OwnedRow::new(vec![
-        Some(1_i32.into()),
-        Some(44_i32.into()),
-        Some(444_i32.into()),
-    ]));
-    state_table.insert(OwnedRow::new(vec![
-        Some(3_i32.into()),
-        Some(1_i32.into()),
-        Some(111_i32.into()),
-    ]));
-
-    // test may_exist with data in memtable (e3), immutable memtable (e2), committed ssts (e1)
-    check_may_exist(&state_table, vec![1, 3, 4, 6], vec![2, 12]).await;
-
-    let e2_res = test_env.storage.seal_and_sync_epoch(e2).await.unwrap();
-
-    // test may_exist with data in memtable (e3), uncommitted ssts (e2), committed ssts (e1)
-    check_may_exist(&state_table, vec![1, 3, 4, 6], vec![2, 12]).await;
-
-    epoch.inc_for_test();
-    state_table.commit(epoch).await.unwrap();
-    let e3 = epoch.prev;
-
-    // test may_exist with data in immutable memtable (e3), uncommitted ssts (e2), committed
-    // ssts (e1)
-    check_may_exist(&state_table, vec![1, 3, 4, 6], vec![2, 12]).await;
-
-    state_table.insert(OwnedRow::new(vec![
-        Some(1_i32.into()),
-        Some(55_i32.into()),
-        Some(555_i32.into()),
-    ]));
-    state_table.insert(OwnedRow::new(vec![
-        Some(2_i32.into()),
-        Some(1_i32.into()),
-        Some(111_i32.into()),
-    ]));
-
-    // test may_exist with data in memtable (e4), immutable memtable (e3), uncommitted ssts
-    // (e2), committed ssts (e1)
-    check_may_exist(&state_table, vec![1, 3, 4, 6], vec![12]).await;
-
-    test_env.meta_client.commit_epoch(e2, e2_res).await.unwrap();
-    test_env.storage.try_wait_epoch_for_test(e2).await;
-
-    epoch.inc_for_test();
-    state_table.commit(epoch).await.unwrap();
-    let e4 = epoch.prev;
-
-    let e3_res = test_env.storage.seal_and_sync_epoch(e3).await.unwrap();
-    let e4_res = test_env.storage.seal_and_sync_epoch(e4).await.unwrap();
-
-    // test may_exist with data in uncommitted ssts (e3, e4), committed ssts (e1, e2, e3, e4)
-    check_may_exist(&state_table, vec![1, 3, 4, 6], vec![12]).await;
-
-    test_env.meta_client.commit_epoch(e3, e3_res).await.unwrap();
-    test_env.storage.try_wait_epoch_for_test(e3).await;
-
-    test_env.meta_client.commit_epoch(e4, e4_res).await.unwrap();
-    test_env.storage.try_wait_epoch_for_test(e4).await;
-
-    // test may_exist with data in committed ssts (e1, e2, e3, e4)
-    check_may_exist(&state_table, vec![1, 3, 4, 6], vec![12]).await;
-}
-
 // After NULL watermark col values are inserted & deleted, they should not appear in the state table
 // cache. Test for apply_batch.
 #[tokio::test]
@@ -1485,6 +1351,9 @@ async fn test_state_table_watermark_cache_ignore_null() {
         WatermarkCacheStateTable::from_table_catalog(&table, test_env.storage.clone(), None).await;
 
     let mut epoch = EpochPair::new_test_epoch(test_epoch(1));
+    test_env
+        .storage
+        .start_epoch(epoch.curr, HashSet::from_iter([TEST_TABLE_ID]));
     state_table.init_epoch(epoch);
 
     let rows = vec![
@@ -1530,9 +1399,12 @@ async fn test_state_table_watermark_cache_ignore_null() {
     assert_eq!(cache.len(), 0);
 
     let watermark = Timestamptz::from_secs(2500).unwrap().to_scalar_value();
-    state_table.update_watermark(watermark, true);
+    state_table.update_watermark(watermark);
 
     epoch.inc_for_test();
+    test_env
+        .storage
+        .start_epoch(epoch.curr, HashSet::from_iter([TEST_TABLE_ID]));
     state_table.commit(epoch).await.unwrap();
 
     let cache = state_table.get_watermark_cache();
@@ -1605,15 +1477,21 @@ async fn test_state_table_watermark_cache_write_chunk() {
         WatermarkCacheStateTable::from_table_catalog(&table, test_env.storage.clone(), None).await;
 
     let mut epoch = EpochPair::new_test_epoch(test_epoch(1));
+    test_env
+        .storage
+        .start_epoch(epoch.curr, HashSet::from_iter([TEST_TABLE_ID]));
     state_table.init_epoch(epoch);
 
     let cache = state_table.get_watermark_cache();
     assert_eq!(cache.len(), 0);
 
     let watermark = Timestamptz::from_secs(0).unwrap().to_scalar_value();
-    state_table.update_watermark(watermark, true);
+    state_table.update_watermark(watermark);
 
     epoch.inc_for_test();
+    test_env
+        .storage
+        .start_epoch(epoch.curr, HashSet::from_iter([TEST_TABLE_ID]));
     state_table.commit(epoch).await.unwrap();
 
     let inserts_1 = vec![
@@ -1720,9 +1598,12 @@ async fn test_state_table_watermark_cache_write_chunk() {
 
     // Should not cleanup anything.
     let watermark = Timestamptz::from_secs(2500).unwrap().to_scalar_value();
-    state_table.update_watermark(watermark, true);
+    state_table.update_watermark(watermark);
 
     epoch.inc_for_test();
+    test_env
+        .storage
+        .start_epoch(epoch.curr, HashSet::from_iter([TEST_TABLE_ID]));
     state_table.commit(epoch).await.unwrap();
 
     // After sync, we should scan all rows into watermark cache.
@@ -1771,6 +1652,9 @@ async fn test_state_table_watermark_cache_refill() {
         WatermarkCacheStateTable::from_table_catalog(&table, test_env.storage.clone(), None).await;
 
     let mut epoch = EpochPair::new_test_epoch(test_epoch(1));
+    test_env
+        .storage
+        .start_epoch(epoch.curr, HashSet::from_iter([TEST_TABLE_ID]));
     state_table.init_epoch(epoch);
 
     let rows = vec![
@@ -1817,9 +1701,12 @@ async fn test_state_table_watermark_cache_refill() {
     assert_eq!(cache.len(), 0);
 
     let watermark = Timestamptz::from_secs(2500).unwrap().to_scalar_value();
-    state_table.update_watermark(watermark, true);
+    state_table.update_watermark(watermark);
 
     epoch.inc_for_test();
+    test_env
+        .storage
+        .start_epoch(epoch.curr, HashSet::from_iter([TEST_TABLE_ID]));
     state_table.commit(epoch).await.unwrap();
 
     // After the first barrier, watermark cache won't be filled.
@@ -1861,6 +1748,9 @@ async fn test_state_table_iter_prefix_and_sub_range() {
         StateTable::from_table_catalog_inconsistent_op(&table, test_env.storage.clone(), None)
             .await;
     let mut epoch = EpochPair::new_test_epoch(test_epoch(1));
+    test_env
+        .storage
+        .start_epoch(epoch.curr, HashSet::from_iter([TEST_TABLE_ID]));
     state_table.init_epoch(epoch);
 
     state_table.insert(OwnedRow::new(vec![
@@ -1886,6 +1776,9 @@ async fn test_state_table_iter_prefix_and_sub_range() {
     ]));
 
     epoch.inc_for_test();
+    test_env
+        .storage
+        .start_epoch(epoch.curr, HashSet::from_iter([TEST_TABLE_ID]));
     state_table.commit(epoch).await.unwrap();
 
     let pk_prefix = OwnedRow::new(vec![Some(1_i32.into())]);
@@ -2056,6 +1949,9 @@ async fn test_replicated_state_table_replication() {
         .await;
 
     let mut epoch = EpochPair::new_test_epoch(test_epoch(1));
+    test_env
+        .storage
+        .start_epoch(epoch.curr, HashSet::from_iter([TEST_TABLE_ID]));
     state_table.init_epoch(epoch);
     replicated_state_table.init_epoch(epoch).await.unwrap();
 
@@ -2067,6 +1963,9 @@ async fn test_replicated_state_table_replication() {
     ]));
 
     epoch.inc_for_test();
+    test_env
+        .storage
+        .start_epoch(epoch.curr, HashSet::from_iter([TEST_TABLE_ID]));
     state_table.commit(epoch).await.unwrap();
     replicated_state_table.commit(epoch).await.unwrap();
     test_env.commit_epoch(epoch.prev).await;
@@ -2127,6 +2026,9 @@ async fn test_replicated_state_table_replication() {
     replicated_state_table.write_chunk(replicate_chunk);
 
     epoch.inc_for_test();
+    test_env
+        .storage
+        .start_epoch(epoch.curr, HashSet::from_iter([TEST_TABLE_ID]));
     state_table.commit(epoch).await.unwrap();
     replicated_state_table.commit(epoch).await.unwrap();
 
diff --git a/src/stream/src/common/table/test_storage_table.rs b/src/stream/src/common/table/test_storage_table.rs
index 098632192d027..1eb552271dced 100644
--- a/src/stream/src/common/table/test_storage_table.rs
+++ b/src/stream/src/common/table/test_storage_table.rs
@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::collections::HashSet;
+
 use futures::{pin_mut, StreamExt};
 use itertools::Itertools;
 use risingwave_common::catalog::{ColumnDesc, ColumnId, TableId};
@@ -77,6 +79,9 @@ async fn test_storage_table_value_indices() {
         value_indices.into_iter().map(|v| v as usize).collect_vec(),
     );
     let mut epoch = EpochPair::new_test_epoch(test_epoch(1));
+    test_env
+        .storage
+        .start_epoch(epoch.curr, HashSet::from_iter([TEST_TABLE_ID]));
     state.init_epoch(epoch);
 
     state.insert(OwnedRow::new(vec![
@@ -110,6 +115,9 @@ async fn test_storage_table_value_indices() {
     ]));
 
     epoch.inc_for_test();
+    test_env
+        .storage
+        .start_epoch(epoch.curr, HashSet::from_iter([TEST_TABLE_ID]));
     state.commit(epoch).await.unwrap();
     test_env.commit_epoch(epoch.prev).await;
 
@@ -197,6 +205,9 @@ async fn test_shuffled_column_id_for_storage_table_get_row() {
             .await;
 
     let mut epoch = EpochPair::new_test_epoch(test_epoch(1));
+    test_env
+        .storage
+        .start_epoch(epoch.curr, HashSet::from_iter([TEST_TABLE_ID]));
     state.init_epoch(epoch);
 
     let table = StorageTable::for_test(
@@ -223,6 +234,9 @@ async fn test_shuffled_column_id_for_storage_table_get_row() {
     ]));
 
     epoch.inc_for_test();
+    test_env
+        .storage
+        .start_epoch(epoch.curr, HashSet::from_iter([TEST_TABLE_ID]));
     state.commit(epoch).await.unwrap();
     test_env.commit_epoch(epoch.prev).await;
 
@@ -310,6 +324,9 @@ async fn test_row_based_storage_table_point_get_in_batch_mode() {
         value_indices,
     );
     let mut epoch = EpochPair::new_test_epoch(test_epoch(1));
+    test_env
+        .storage
+        .start_epoch(epoch.curr, HashSet::from_iter([TEST_TABLE_ID]));
     state.init_epoch(epoch);
 
     state.insert(OwnedRow::new(vec![Some(1_i32.into()), None, None]));
@@ -326,6 +343,9 @@ async fn test_row_based_storage_table_point_get_in_batch_mode() {
         Some(222_i32.into()),
     ]));
     epoch.inc_for_test();
+    test_env
+        .storage
+        .start_epoch(epoch.curr, HashSet::from_iter([TEST_TABLE_ID]));
     state.commit(epoch).await.unwrap();
     test_env.commit_epoch(epoch.prev).await;
 
@@ -415,6 +435,9 @@ async fn test_batch_scan_with_value_indices() {
         value_indices,
     );
     let mut epoch = EpochPair::new_test_epoch(test_epoch(1));
+    test_env
+        .storage
+        .start_epoch(epoch.curr, HashSet::from_iter([TEST_TABLE_ID]));
     state.init_epoch(epoch);
 
     state.insert(OwnedRow::new(vec![
@@ -437,6 +460,9 @@ async fn test_batch_scan_with_value_indices() {
     ]));
 
     epoch.inc_for_test();
+    test_env
+        .storage
+        .start_epoch(epoch.curr, HashSet::from_iter([TEST_TABLE_ID]));
     state.commit(epoch).await.unwrap();
     test_env.commit_epoch(epoch.prev).await;
 
@@ -513,6 +539,9 @@ async fn test_batch_scan_chunk_with_value_indices() {
         value_indices.clone(),
     );
     let mut epoch = EpochPair::new_test_epoch(test_epoch(1));
+    test_env
+        .storage
+        .start_epoch(epoch.curr, HashSet::from_iter([TEST_TABLE_ID]));
     state.init_epoch(epoch);
 
     let gen_row = |i: i32, is_update: bool| {
@@ -554,6 +583,9 @@ async fn test_batch_scan_chunk_with_value_indices() {
         .collect_vec();
 
     epoch.inc_for_test();
+    test_env
+        .storage
+        .start_epoch(epoch.curr, HashSet::from_iter([TEST_TABLE_ID]));
     state.commit(epoch).await.unwrap();
     test_env.commit_epoch(epoch.prev).await;
 
diff --git a/src/stream/src/common/table/watermark.rs b/src/stream/src/common/table/watermark.rs
deleted file mode 100644
index 3e618873ec65b..0000000000000
--- a/src/stream/src/common/table/watermark.rs
+++ /dev/null
@@ -1,60 +0,0 @@
-// Copyright 2024 RisingWave Labs
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-/// Strategy to decide how to buffer the watermarks, used for state cleaning.
-pub trait WatermarkBufferStrategy: Default {
-    /// Trigger when a epoch is committed.
-    fn tick(&mut self);
-
-    /// Whether to clear the buffer.
-    ///
-    /// Returns true to indicate that the buffer should be cleared and the strategy states reset.
-    fn apply(&mut self) -> bool;
-}
-
-/// No buffer, apply watermark to memory immediately.
-/// Use the strategy when you want to apply the watermark immediately.
-#[derive(Default, Debug)]
-pub struct WatermarkNoBuffer;
-
-impl WatermarkBufferStrategy for WatermarkNoBuffer {
-    fn tick(&mut self) {}
-
-    fn apply(&mut self) -> bool {
-        true
-    }
-}
-
-/// Buffer the watermark by a epoch period.
-/// The strategy reduced the delete-range calls to storage.
-#[derive(Default, Debug)]
-pub struct WatermarkBufferByEpoch<const PERIOD: usize> {
-    /// number of epochs since the last time we did state cleaning by watermark.
-    buffered_epochs_cnt: usize,
-}
-
-impl<const PERIOD: usize> WatermarkBufferStrategy for WatermarkBufferByEpoch<PERIOD> {
-    fn tick(&mut self) {
-        self.buffered_epochs_cnt += 1;
-    }
-
-    fn apply(&mut self) -> bool {
-        if self.buffered_epochs_cnt >= PERIOD {
-            self.buffered_epochs_cnt = 0;
-            true
-        } else {
-            false
-        }
-    }
-}
diff --git a/src/stream/src/error.rs b/src/stream/src/error.rs
index defefa7b474ae..725e7706c9fb6 100644
--- a/src/stream/src/error.rs
+++ b/src/stream/src/error.rs
@@ -13,8 +13,8 @@
 // limitations under the License.
 
 use risingwave_common::array::ArrayError;
+use risingwave_common::secret::SecretError;
 use risingwave_connector::error::ConnectorError;
-use risingwave_connector::sink::SinkError;
 use risingwave_expr::ExprError;
 use risingwave_pb::PbFieldNotFound;
 use risingwave_rpc_client::error::ToTonicStatus;
@@ -65,13 +65,6 @@ pub enum ErrorKind {
         StreamExecutorError,
     ),
 
-    #[error("Sink error: {0}")]
-    Sink(
-        #[from]
-        #[backtrace]
-        SinkError,
-    ),
-
     #[error("Actor {actor_id} exited unexpectedly: {source}")]
     UnexpectedExit {
         actor_id: ActorId,
@@ -86,6 +79,13 @@ pub enum ErrorKind {
         reason: &'static str,
     },
 
+    #[error("Secret error: {0}")]
+    Secret(
+        #[from]
+        #[backtrace]
+        SecretError,
+    ),
+
     #[error(transparent)]
     Uncategorized(
         #[from]
diff --git a/src/stream/src/executor/aggregation/agg_group.rs b/src/stream/src/executor/aggregation/agg_group.rs
index 325a9001811e6..9a37507370289 100644
--- a/src/stream/src/executor/aggregation/agg_group.rs
+++ b/src/stream/src/executor/aggregation/agg_group.rs
@@ -18,7 +18,7 @@ use std::sync::Arc;
 
 use risingwave_common::array::stream_record::{Record, RecordType};
 use risingwave_common::array::StreamChunk;
-use risingwave_common::buffer::Bitmap;
+use risingwave_common::bitmap::Bitmap;
 use risingwave_common::catalog::Schema;
 use risingwave_common::must_match;
 use risingwave_common::row::{OwnedRow, Row, RowExt};
@@ -54,9 +54,9 @@ pub struct OnlyOutputIfHasInput;
 impl Strategy for AlwaysOutput {
     fn infer_change_type(
         prev_row_count: usize,
-        curr_row_count: usize,
+        _curr_row_count: usize,
         prev_outputs: Option<&OwnedRow>,
-        curr_outputs: &OwnedRow,
+        _curr_outputs: &OwnedRow,
     ) -> Option<RecordType> {
         match prev_outputs {
             None => {
@@ -68,14 +68,17 @@ impl Strategy for AlwaysOutput {
                 // Generate output no matter whether current row count is 0 or not.
                 Some(RecordType::Insert)
             }
-            Some(prev_outputs) => {
-                if prev_row_count == 0 && curr_row_count == 0 || prev_outputs == curr_outputs {
-                    // No rows exist, or output is not changed.
-                    None
-                } else {
-                    Some(RecordType::Update)
-                }
-            }
+            // NOTE(kwannoel): We always output, even if the update is a no-op.
+            // e.g. the following will still be emitted downstream:
+            // ```
+            // U- 1
+            // U+ 1
+            // ```
+            // This is to support `approx_percentile` via `row_merge`, which requires
+            // both the lhs and rhs to always output updates per epoch, or not all.
+            // Otherwise we are unable to construct a full row, if only one side updates,
+            // as the `row_merge` executor is stateless.
+            Some(_prev_outputs) => Some(RecordType::Update),
         }
     }
 }
diff --git a/src/stream/src/executor/aggregation/agg_state.rs b/src/stream/src/executor/aggregation/agg_state.rs
index 6ef55fcedd5ed..a0fe4ffa97fe6 100644
--- a/src/stream/src/executor/aggregation/agg_state.rs
+++ b/src/stream/src/executor/aggregation/agg_state.rs
@@ -13,7 +13,7 @@
 // limitations under the License.
 
 use risingwave_common::array::StreamChunk;
-use risingwave_common::buffer::Bitmap;
+use risingwave_common::bitmap::Bitmap;
 use risingwave_common::catalog::Schema;
 use risingwave_common::must_match;
 use risingwave_common::types::Datum;
diff --git a/src/stream/src/executor/aggregation/agg_state_cache.rs b/src/stream/src/executor/aggregation/agg_state_cache.rs
index 71cd9a45e3f1b..8a202ed6dd1dd 100644
--- a/src/stream/src/executor/aggregation/agg_state_cache.rs
+++ b/src/stream/src/executor/aggregation/agg_state_cache.rs
@@ -23,7 +23,7 @@ use risingwave_common::util::row_serde::OrderedRowSerde;
 use risingwave_common_estimate_size::EstimateSize;
 use smallvec::SmallVec;
 
-use crate::common::cache::{StateCache, StateCacheFiller};
+use crate::common::state_cache::{StateCache, StateCacheFiller};
 
 /// Cache key type.
 type CacheKey = MemcmpEncoded;
diff --git a/src/stream/src/executor/aggregation/distinct.rs b/src/stream/src/executor/aggregation/distinct.rs
index 3433708957aba..ee54f524e592d 100644
--- a/src/stream/src/executor/aggregation/distinct.rs
+++ b/src/stream/src/executor/aggregation/distinct.rs
@@ -18,7 +18,7 @@ use std::sync::atomic::AtomicU64;
 
 use itertools::Itertools;
 use risingwave_common::array::{ArrayRef, Op};
-use risingwave_common::buffer::{Bitmap, BitmapBuilder};
+use risingwave_common::bitmap::{Bitmap, BitmapBuilder};
 use risingwave_common::row::{self, CompactedRow, RowExt};
 use risingwave_common::util::iter_util::ZipEqFast;
 
diff --git a/src/stream/src/executor/aggregation/minput.rs b/src/stream/src/executor/aggregation/minput.rs
index c8bae26f6e1b3..b8df965ab84ed 100644
--- a/src/stream/src/executor/aggregation/minput.rs
+++ b/src/stream/src/executor/aggregation/minput.rs
@@ -24,14 +24,14 @@ use risingwave_common::types::Datum;
 use risingwave_common::util::row_serde::OrderedRowSerde;
 use risingwave_common::util::sort_util::{ColumnOrder, OrderType};
 use risingwave_common_estimate_size::EstimateSize;
-use risingwave_expr::aggregate::{AggCall, AggKind, BoxedAggregateFunction};
+use risingwave_expr::aggregate::{AggCall, AggKind, BoxedAggregateFunction, PbAggKind};
 use risingwave_pb::stream_plan::PbAggNodeVersion;
 use risingwave_storage::store::PrefetchOptions;
 use risingwave_storage::StateStore;
 
 use super::agg_state_cache::{AggStateCache, GenericAggStateCache};
 use super::GroupKey;
-use crate::common::cache::{OrderedStateCache, TopNStateCache};
+use crate::common::state_cache::{OrderedStateCache, TopNStateCache};
 use crate::common::table::state_table::StateTable;
 use crate::common::StateTableColumnMapping;
 use crate::executor::{PkIndices, StreamExecutorResult};
@@ -125,16 +125,18 @@ impl MaterializedInputState {
         let cache_key_serializer = OrderedRowSerde::new(cache_key_data_types, order_types);
 
         let cache: Box<dyn AggStateCache + Send + Sync> = match agg_call.kind {
-            AggKind::Min | AggKind::Max | AggKind::FirstValue | AggKind::LastValue => {
-                Box::new(GenericAggStateCache::new(
-                    TopNStateCache::new(extreme_cache_size),
-                    agg_call.args.arg_types(),
-                ))
-            }
-            AggKind::StringAgg
-            | AggKind::ArrayAgg
-            | AggKind::JsonbAgg
-            | AggKind::JsonbObjectAgg => Box::new(GenericAggStateCache::new(
+            AggKind::Builtin(
+                PbAggKind::Min | PbAggKind::Max | PbAggKind::FirstValue | PbAggKind::LastValue,
+            ) => Box::new(GenericAggStateCache::new(
+                TopNStateCache::new(extreme_cache_size),
+                agg_call.args.arg_types(),
+            )),
+            AggKind::Builtin(
+                PbAggKind::StringAgg
+                | PbAggKind::ArrayAgg
+                | PbAggKind::JsonbAgg
+                | PbAggKind::JsonbObjectAgg,
+            ) => Box::new(GenericAggStateCache::new(
                 OrderedStateCache::new(),
                 agg_call.args.arg_types(),
             )),
@@ -145,7 +147,9 @@ impl MaterializedInputState {
         };
         let output_first_value = matches!(
             agg_call.kind,
-            AggKind::Min | AggKind::Max | AggKind::FirstValue | AggKind::LastValue
+            AggKind::Builtin(
+                PbAggKind::Min | PbAggKind::Max | PbAggKind::FirstValue | PbAggKind::LastValue
+            )
         );
 
         Ok(Self {
@@ -239,32 +243,34 @@ fn generate_order_columns_before_version_issue_13465(
     pk_indices: &PkIndices,
     arg_col_indices: &[usize],
 ) -> (Vec<usize>, Vec<OrderType>) {
-    let (mut order_col_indices, mut order_types) =
-        if matches!(agg_call.kind, AggKind::Min | AggKind::Max) {
-            // `min`/`max` need not to order by any other columns, but have to
-            // order by the agg value implicitly.
-            let order_type = if agg_call.kind == AggKind::Min {
-                OrderType::ascending()
-            } else {
-                OrderType::descending()
-            };
-            (vec![arg_col_indices[0]], vec![order_type])
+    let (mut order_col_indices, mut order_types) = if matches!(
+        agg_call.kind,
+        AggKind::Builtin(PbAggKind::Min | PbAggKind::Max)
+    ) {
+        // `min`/`max` need not to order by any other columns, but have to
+        // order by the agg value implicitly.
+        let order_type = if matches!(agg_call.kind, AggKind::Builtin(PbAggKind::Min)) {
+            OrderType::ascending()
         } else {
-            agg_call
-                .column_orders
-                .iter()
-                .map(|p| {
-                    (
-                        p.column_index,
-                        if agg_call.kind == AggKind::LastValue {
-                            p.order_type.reverse()
-                        } else {
-                            p.order_type
-                        },
-                    )
-                })
-                .unzip()
+            OrderType::descending()
         };
+        (vec![arg_col_indices[0]], vec![order_type])
+    } else {
+        agg_call
+            .column_orders
+            .iter()
+            .map(|p| {
+                (
+                    p.column_index,
+                    if matches!(agg_call.kind, AggKind::Builtin(PbAggKind::LastValue)) {
+                        p.order_type.reverse()
+                    } else {
+                        p.order_type
+                    },
+                )
+            })
+            .unzip()
+    };
 
     if agg_call.distinct {
         // If distinct, we need to materialize input with the distinct keys
diff --git a/src/stream/src/executor/aggregation/mod.rs b/src/stream/src/executor/aggregation/mod.rs
index 619a2bc8df527..a5bd631e0caec 100644
--- a/src/stream/src/executor/aggregation/mod.rs
+++ b/src/stream/src/executor/aggregation/mod.rs
@@ -18,8 +18,8 @@ pub use distinct::*;
 use risingwave_common::array::ArrayImpl::Bool;
 use risingwave_common::array::DataChunk;
 use risingwave_common::bail;
-use risingwave_common::buffer::Bitmap;
-use risingwave_expr::aggregate::{AggCall, AggKind};
+use risingwave_common::bitmap::Bitmap;
+use risingwave_expr::aggregate::{AggCall, AggKind, PbAggKind};
 use risingwave_expr::expr::{LogReport, NonStrictExpression};
 use risingwave_storage::StateStore;
 
@@ -39,7 +39,7 @@ pub async fn agg_call_filter_res(
     let mut vis = chunk.visibility().clone();
     if matches!(
         agg_call.kind,
-        AggKind::Min | AggKind::Max | AggKind::StringAgg
+        AggKind::Builtin(PbAggKind::Min | PbAggKind::Max | PbAggKind::StringAgg)
     ) {
         // should skip NULL value for these kinds of agg function
         let agg_col_idx = agg_call.args.val_indices()[0]; // the first arg is the agg column for all these kinds
diff --git a/src/stream/src/executor/approx_percentile/global.rs b/src/stream/src/executor/approx_percentile/global.rs
new file mode 100644
index 0000000000000..534857237bffb
--- /dev/null
+++ b/src/stream/src/executor/approx_percentile/global.rs
@@ -0,0 +1,244 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use core::ops::Bound;
+
+use risingwave_common::array::Op;
+use risingwave_common::row::RowExt;
+use risingwave_common::types::ToOwnedDatum;
+use risingwave_storage::store::PrefetchOptions;
+
+use crate::executor::prelude::*;
+
+pub struct GlobalApproxPercentileExecutor<S: StateStore> {
+    _ctx: ActorContextRef,
+    pub input: Executor,
+    pub quantile: f64,
+    pub base: f64,
+    pub chunk_size: usize,
+    /// Used for the approx percentile buckets.
+    pub bucket_state_table: StateTable<S>,
+    /// Used for the approx percentile count.
+    pub count_state_table: StateTable<S>,
+}
+
+impl<S: StateStore> GlobalApproxPercentileExecutor<S> {
+    pub fn new(
+        _ctx: ActorContextRef,
+        input: Executor,
+        quantile: f64,
+        base: f64,
+        chunk_size: usize,
+        bucket_state_table: StateTable<S>,
+        count_state_table: StateTable<S>,
+    ) -> Self {
+        Self {
+            _ctx,
+            input,
+            quantile,
+            base,
+            chunk_size,
+            bucket_state_table,
+            count_state_table,
+        }
+    }
+
+    /// TODO(kwannoel): Include cache later.
+    #[try_stream(ok = Message, error = StreamExecutorError)]
+    async fn execute_inner(self) {
+        let mut bucket_state_table = self.bucket_state_table;
+        let mut count_state_table = self.count_state_table;
+        let mut input_stream = self.input.execute();
+
+        // Initialize state tables.
+        let first_barrier = expect_first_barrier(&mut input_stream).await?;
+        bucket_state_table.init_epoch(first_barrier.epoch);
+        count_state_table.init_epoch(first_barrier.epoch);
+        yield Message::Barrier(first_barrier);
+
+        // Get row count state, and row_count.
+        let mut row_count_state = count_state_table.get_row(&[Datum::None; 0]).await?;
+        let mut row_count = if let Some(row) = row_count_state.as_ref() {
+            row.datum_at(0).unwrap().into_int64()
+        } else {
+            0
+        };
+
+        // Get prev output, based on the current state.
+        let mut prev_output = Self::get_output(
+            &bucket_state_table,
+            row_count as u64,
+            self.quantile,
+            self.base,
+        )
+        .await?;
+
+        let mut received_input = false;
+
+        #[for_await]
+        for message in input_stream {
+            match message? {
+                Message::Chunk(chunk) => {
+                    received_input = true;
+                    for (_, row) in chunk.rows() {
+                        // Decoding
+                        let sign_datum = row.datum_at(0);
+                        let bucket_id_datum = row.datum_at(1);
+                        let delta_datum = row.datum_at(2);
+                        let delta: i32 = delta_datum.unwrap().into_int32();
+
+                        // Updates
+                        row_count = row_count.checked_add(delta as i64).unwrap();
+
+                        let pk = row.project(&[0, 1]);
+                        let old_row = bucket_state_table.get_row(pk).await?;
+                        let old_bucket_row_count: i64 = if let Some(row) = old_row.as_ref() {
+                            row.datum_at(2).unwrap().into_int64()
+                        } else {
+                            0
+                        };
+
+                        let new_value = old_bucket_row_count.checked_add(delta as i64).unwrap();
+                        let new_value_datum = Datum::from(ScalarImpl::Int64(new_value));
+                        let new_row = &[
+                            sign_datum.to_owned_datum(),
+                            bucket_id_datum.map(|d| d.into()),
+                            new_value_datum,
+                        ];
+
+                        if old_row.is_none() {
+                            bucket_state_table.insert(new_row);
+                        } else {
+                            bucket_state_table.update(old_row, new_row);
+                        }
+                    }
+                }
+                Message::Barrier(barrier) => {
+                    // If we haven't received any input, we don't need to update the state.
+                    // This is unless row count state is empty, then we need to persist the state,
+                    // and yield a NULL downstream.
+                    if !received_input && row_count_state.is_some() {
+                        count_state_table.commit(barrier.epoch).await?;
+                        bucket_state_table.commit(barrier.epoch).await?;
+                        yield Message::Barrier(barrier);
+                        continue;
+                    }
+                    // We maintain an invariant, iff row_count_state is none,
+                    // we haven't pushed any data to downstream.
+                    // Naturally, if row_count_state is some,
+                    // we have pushed data to downstream.
+                    let new_output = Self::get_output(
+                        &bucket_state_table,
+                        row_count as u64,
+                        self.quantile,
+                        self.base,
+                    )
+                    .await?;
+                    let percentile_chunk = if row_count_state.is_none() {
+                        StreamChunk::from_rows(
+                            &[(Op::Insert, &[new_output.clone()])],
+                            &[DataType::Float64],
+                        )
+                    } else {
+                        StreamChunk::from_rows(
+                            &[
+                                (Op::UpdateDelete, &[prev_output.clone()]),
+                                (Op::UpdateInsert, &[new_output.clone()]),
+                            ],
+                            &[DataType::Float64],
+                        )
+                    };
+                    prev_output = new_output;
+                    yield Message::Chunk(percentile_chunk);
+
+                    let new_row_count_state = &[Datum::from(ScalarImpl::Int64(row_count))];
+                    if let Some(row_count_state) = row_count_state {
+                        count_state_table.update(row_count_state, new_row_count_state);
+                    } else {
+                        count_state_table.insert(new_row_count_state);
+                    }
+                    row_count_state = Some(new_row_count_state.into_owned_row());
+                    count_state_table.commit(barrier.epoch).await?;
+
+                    bucket_state_table.commit(barrier.epoch).await?;
+
+                    yield Message::Barrier(barrier);
+                }
+                Message::Watermark(_) => {}
+            }
+        }
+    }
+
+    /// We have these scenarios to consider, based on row count state.
+    /// 1. We have no row count state, this means it's the bootstrap init for this executor.
+    ///    Output NULL as an INSERT. Persist row count state=0.
+    /// 2. We have row count state.
+    ///    Output UPDATE (`old_state`, `new_state`) to downstream.
+    async fn get_output(
+        bucket_state_table: &StateTable<S>,
+        row_count: u64,
+        quantile: f64,
+        base: f64,
+    ) -> StreamExecutorResult<Datum> {
+        let quantile_count = (row_count as f64 * quantile).floor() as u64;
+        let mut acc_count = 0;
+        let neg_bounds: (Bound<OwnedRow>, Bound<OwnedRow>) = (
+            Bound::Unbounded,
+            Bound::Excluded([Datum::from(ScalarImpl::Int16(0))].to_owned_row()),
+        );
+        let non_neg_bounds: (Bound<OwnedRow>, Bound<OwnedRow>) = (
+            Bound::Included([Datum::from(ScalarImpl::Int16(0))].to_owned_row()),
+            Bound::Unbounded,
+        );
+        // Just iterate over the singleton vnode.
+        // TODO(kwannoel): Should we just use separate state tables for
+        // positive and negative counts?
+        // Reverse iterator is not as efficient.
+        #[for_await]
+        for keyed_row in bucket_state_table
+            .rev_iter_with_prefix(&[Datum::None; 0], &neg_bounds, PrefetchOptions::default())
+            .await?
+            .chain(
+                bucket_state_table
+                    .iter_with_prefix(
+                        &[Datum::None; 0],
+                        &non_neg_bounds,
+                        PrefetchOptions::default(),
+                    )
+                    .await?,
+            )
+        {
+            let row = keyed_row?.into_owned_row();
+            let count = row.datum_at(2).unwrap().into_int64();
+            acc_count += count as u64;
+            if acc_count > quantile_count {
+                let sign = row.datum_at(0).unwrap().into_int16();
+                if sign == 0 {
+                    return Ok(Datum::from(ScalarImpl::Float64(0.0.into())));
+                }
+                let bucket_id = row.datum_at(1).unwrap().into_int32();
+                let percentile_value = sign as f64 * 2.0 * base.powi(bucket_id) / (base + 1.0);
+                let percentile_datum = Datum::from(ScalarImpl::Float64(percentile_value.into()));
+                return Ok(percentile_datum);
+            }
+        }
+        Ok(Datum::None)
+    }
+}
+
+impl<S: StateStore> Execute for GlobalApproxPercentileExecutor<S> {
+    fn execute(self: Box<Self>) -> BoxedMessageStream {
+        self.execute_inner().boxed()
+    }
+}
diff --git a/src/stream/src/executor/approx_percentile/local.rs b/src/stream/src/executor/approx_percentile/local.rs
new file mode 100644
index 0000000000000..b50419342c0d7
--- /dev/null
+++ b/src/stream/src/executor/approx_percentile/local.rs
@@ -0,0 +1,126 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::HashMap;
+use std::iter;
+
+use risingwave_common::array::Op;
+use risingwave_common::util::chunk_coalesce::DataChunkBuilder;
+
+use crate::executor::prelude::*;
+
+pub struct LocalApproxPercentileExecutor {
+    _ctx: ActorContextRef,
+    pub input: Executor,
+    pub base: f64,
+    pub percentile_index: usize,
+    pub chunk_size: usize,
+}
+
+impl LocalApproxPercentileExecutor {
+    pub fn new(
+        _ctx: ActorContextRef,
+        input: Executor,
+        base: f64,
+        percentile_index: usize,
+        chunk_size: usize,
+    ) -> Self {
+        Self {
+            _ctx,
+            input,
+            base,
+            percentile_index,
+            chunk_size,
+        }
+    }
+
+    #[try_stream(ok = Message, error = StreamExecutorError)]
+    async fn execute_inner(self) {
+        let percentile_index = self.percentile_index;
+        #[for_await]
+        for message in self.input.execute() {
+            match message? {
+                Message::Chunk(chunk) => {
+                    let mut builder = DataChunkBuilder::new(
+                        vec![DataType::Int16, DataType::Int32, DataType::Int32],
+                        self.chunk_size,
+                    );
+                    let chunk = chunk.project(&[percentile_index]);
+                    let mut pos_counts = HashMap::new();
+                    let mut neg_counts = HashMap::new();
+                    let mut zero_count = 0;
+                    for (op, row) in chunk.rows() {
+                        let value = row.datum_at(0).unwrap();
+                        let value: f64 = value.into_float64().into_inner();
+                        if value < 0.0 {
+                            let value = -value;
+                            let bucket = value.log(self.base).ceil() as i32; // TODO(kwannoel): should this be floor??
+                            let count = neg_counts.entry(bucket).or_insert(0);
+                            match op {
+                                Op::Insert | Op::UpdateInsert => *count += 1,
+                                Op::Delete | Op::UpdateDelete => *count -= 1,
+                            }
+                        } else if value > 0.0 {
+                            let bucket = value.log(self.base).ceil() as i32;
+                            let count = pos_counts.entry(bucket).or_insert(0);
+                            match op {
+                                Op::Insert | Op::UpdateInsert => *count += 1,
+                                Op::Delete | Op::UpdateDelete => *count -= 1,
+                            }
+                        } else {
+                            match op {
+                                Op::Insert | Op::UpdateInsert => zero_count += 1,
+                                Op::Delete | Op::UpdateDelete => zero_count -= 1,
+                            }
+                        }
+                    }
+
+                    for (sign, bucket, count) in neg_counts
+                        .into_iter()
+                        .map(|(b, c)| (-1, b, c))
+                        .chain(pos_counts.into_iter().map(|(b, c)| (1, b, c)))
+                        .chain(iter::once((0, 0, zero_count)))
+                    {
+                        let row = [
+                            Datum::from(ScalarImpl::Int16(sign)),
+                            Datum::from(ScalarImpl::Int32(bucket)),
+                            Datum::from(ScalarImpl::Int32(count)),
+                        ];
+                        if let Some(data_chunk) = builder.append_one_row(&row) {
+                            // NOTE(kwannoel): The op here is simply ignored.
+                            // The downstream global_approx_percentile will always just update its bucket counts.
+                            let ops = vec![Op::Insert; data_chunk.cardinality()];
+                            let chunk = StreamChunk::from_parts(ops, data_chunk);
+                            yield Message::Chunk(chunk);
+                        }
+                    }
+                    if !builder.is_empty() {
+                        let data_chunk = builder.finish();
+                        let ops = vec![Op::Insert; data_chunk.cardinality()];
+                        let chunk = StreamChunk::from_parts(ops, data_chunk);
+                        yield Message::Chunk(chunk);
+                    }
+                }
+                b @ Message::Barrier(_) => yield b,
+                Message::Watermark(_) => {}
+            }
+        }
+    }
+}
+
+impl Execute for LocalApproxPercentileExecutor {
+    fn execute(self: Box<Self>) -> BoxedMessageStream {
+        self.execute_inner().boxed()
+    }
+}
diff --git a/src/stream/src/executor/approx_percentile/mod.rs b/src/stream/src/executor/approx_percentile/mod.rs
new file mode 100644
index 0000000000000..29910d9032e15
--- /dev/null
+++ b/src/stream/src/executor/approx_percentile/mod.rs
@@ -0,0 +1,16 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub mod global;
+pub mod local;
diff --git a/src/stream/src/executor/backfill/arrangement_backfill.rs b/src/stream/src/executor/backfill/arrangement_backfill.rs
index 7920e8dceee80..e3979496731b5 100644
--- a/src/stream/src/executor/backfill/arrangement_backfill.rs
+++ b/src/stream/src/executor/backfill/arrangement_backfill.rs
@@ -462,7 +462,7 @@ where
                 // May need to revisit it.
                 // Need to check it after scale-in / scale-out.
                 self.progress.update(
-                    barrier.epoch.curr,
+                    barrier.epoch,
                     snapshot_read_epoch,
                     total_snapshot_processed_rows,
                 );
@@ -577,7 +577,7 @@ where
                     }
 
                     self.progress
-                        .finish(barrier.epoch.curr, total_snapshot_processed_rows);
+                        .finish(barrier.epoch, total_snapshot_processed_rows);
                     yield msg;
                     break;
                 }
diff --git a/src/stream/src/executor/backfill/cdc/cdc_backfill.rs b/src/stream/src/executor/backfill/cdc/cdc_backfill.rs
index 4e564130accbb..943059355f054 100644
--- a/src/stream/src/executor/backfill/cdc/cdc_backfill.rs
+++ b/src/stream/src/executor/backfill/cdc/cdc_backfill.rs
@@ -42,6 +42,7 @@ use crate::executor::backfill::utils::{
 use crate::executor::backfill::CdcScanOptions;
 use crate::executor::monitor::CdcBackfillMetrics;
 use crate::executor::prelude::*;
+use crate::executor::UpdateMutation;
 use crate::task::CreateMviewProgress;
 
 /// `split_id`, `is_finished`, `row_count`, `cdc_offset` all occupy 1 column each.
@@ -135,7 +136,7 @@ impl<S: StateStore> CdcBackfillExecutor<S> {
         let pk_indices = self.external_table.pk_indices().to_vec();
         let pk_order = self.external_table.pk_order_types().to_vec();
 
-        let upstream_table_id = self.external_table.table_id().table_id;
+        let table_id = self.external_table.table_id().table_id;
         let upstream_table_name = self.external_table.qualified_table_name();
         let schema_table_name = self.external_table.schema_table_name().clone();
         let external_database_name = self.external_table.database_name().to_owned();
@@ -157,7 +158,7 @@ impl<S: StateStore> CdcBackfillExecutor<S> {
         // Poll the upstream to get the first barrier.
         let first_barrier = expect_first_barrier(&mut upstream).await?;
 
-        let mut paused = first_barrier.is_pause_on_startup();
+        let mut is_snapshot_paused = first_barrier.is_pause_on_startup();
 
         // Check whether this parallelism has been assigned splits,
         // if not, we should bypass the backfill directly.
@@ -192,11 +193,12 @@ impl<S: StateStore> CdcBackfillExecutor<S> {
         let mut consumed_binlog_offset: Option<CdcOffset> = None;
 
         tracing::info!(
-            upstream_table_id,
+            table_id,
             upstream_table_name,
             initial_binlog_offset = ?last_binlog_offset,
             ?current_pk_pos,
             is_finished = state.is_finished,
+            is_snapshot_paused,
             snapshot_row_count = total_snapshot_row_count,
             rate_limit = self.rate_limit_rps,
             disable_backfill = self.options.disable_backfill,
@@ -234,6 +236,27 @@ impl<S: StateStore> CdcBackfillExecutor<S> {
             for msg in upstream.by_ref() {
                 match msg? {
                     Message::Barrier(barrier) => {
+                        match barrier.mutation.as_deref() {
+                            Some(crate::executor::Mutation::Pause) => {
+                                is_snapshot_paused = true;
+                                tracing::info!(
+                                    table_id,
+                                    upstream_table_name,
+                                    "snapshot is paused by barrier"
+                                );
+                            }
+                            Some(crate::executor::Mutation::Resume) => {
+                                is_snapshot_paused = false;
+                                tracing::info!(
+                                    table_id,
+                                    upstream_table_name,
+                                    "snapshot is resumed by barrier"
+                                );
+                            }
+                            _ => {
+                                // ignore other mutations
+                            }
+                        }
                         // commit state just to bump the epoch of state table
                         state_impl.commit_state(barrier.epoch).await?;
                         yield Message::Barrier(barrier);
@@ -248,10 +271,11 @@ impl<S: StateStore> CdcBackfillExecutor<S> {
                 }
             }
 
-            tracing::info!(upstream_table_id,
+            tracing::info!(table_id,
                 upstream_table_name,
                 initial_binlog_offset = ?last_binlog_offset,
                 ?current_pk_pos,
+                is_snapshot_paused,
                 "start cdc backfill loop");
 
             // the buffer will be drained when a barrier comes
@@ -274,9 +298,9 @@ impl<S: StateStore> CdcBackfillExecutor<S> {
                     .snapshot_read_full_table(read_args, self.options.snapshot_batch_size)
                     .map(Either::Right));
 
-                let (right_snapshot, valve) = pausable(right_snapshot);
-                if paused {
-                    valve.pause();
+                let (right_snapshot, snapshot_valve) = pausable(right_snapshot);
+                if is_snapshot_paused {
+                    snapshot_valve.pause();
                 }
 
                 // Prefer to select upstream, so we can stop snapshot stream when barrier comes.
@@ -306,12 +330,12 @@ impl<S: StateStore> CdcBackfillExecutor<S> {
                                         use crate::executor::Mutation;
                                         match mutation {
                                             Mutation::Pause => {
-                                                paused = true;
-                                                valve.pause();
+                                                is_snapshot_paused = true;
+                                                snapshot_valve.pause();
                                             }
                                             Mutation::Resume => {
-                                                paused = false;
-                                                valve.resume();
+                                                is_snapshot_paused = false;
+                                                snapshot_valve.resume();
                                             }
                                             Mutation::Throttle(some) => {
                                                 if let Some(new_rate_limit) =
@@ -323,6 +347,21 @@ impl<S: StateStore> CdcBackfillExecutor<S> {
                                                     continue 'backfill_loop;
                                                 }
                                             }
+                                            Mutation::Update(UpdateMutation {
+                                                dropped_actors,
+                                                ..
+                                            }) => {
+                                                if dropped_actors.contains(&self.actor_ctx.id) {
+                                                    // the actor has been dropped, exit the backfill loop
+                                                    tracing::info!(
+                                                        table_id,
+                                                        upstream_table_name,
+                                                        "CdcBackfill has been dropped due to config change"
+                                                    );
+                                                    yield Message::Barrier(barrier);
+                                                    break 'backfill_loop;
+                                                }
+                                            }
                                             _ => (),
                                         }
                                     }
@@ -339,7 +378,7 @@ impl<S: StateStore> CdcBackfillExecutor<S> {
                                         // staging the barrier
                                         pending_barrier = Some(barrier);
                                         tracing::debug!(
-                                            upstream_table_id,
+                                            table_id,
                                             ?current_pk_pos,
                                             ?snapshot_read_row_cnt,
                                             "Prepare to start a new snapshot"
@@ -406,7 +445,7 @@ impl<S: StateStore> CdcBackfillExecutor<S> {
                             match msg? {
                                 None => {
                                     tracing::info!(
-                                        upstream_table_id,
+                                        table_id,
                                         ?last_binlog_offset,
                                         ?current_pk_pos,
                                         "snapshot read stream ends"
@@ -457,14 +496,20 @@ impl<S: StateStore> CdcBackfillExecutor<S> {
                 // Otherwise, the result set of the new snapshot stream may become empty.
                 // It maybe a cancellation bug of the mysql driver.
                 let (_, mut snapshot_stream) = backfill_stream.into_inner();
-                if let Some(msg) = snapshot_stream.next().await {
+
+                if !is_snapshot_paused
+                    && let Some(msg) = snapshot_stream
+                        .next()
+                        .instrument_await("consume_snapshot_stream_once")
+                        .await
+                {
                     let Either::Right(msg) = msg else {
                         bail!("BUG: snapshot_read contains upstream messages");
                     };
                     match msg? {
                         None => {
                             tracing::info!(
-                                upstream_table_id,
+                                table_id,
                                 ?last_binlog_offset,
                                 ?current_pk_pos,
                                 "snapshot read stream ends in the force emit branch"
@@ -501,7 +546,7 @@ impl<S: StateStore> CdcBackfillExecutor<S> {
                             snapshot_read_row_cnt += row_count as usize;
 
                             tracing::debug!(
-                                upstream_table_id,
+                                table_id,
                                 ?current_pk_pos,
                                 ?snapshot_read_row_cnt,
                                 "force emit a snapshot chunk"
@@ -567,7 +612,7 @@ impl<S: StateStore> CdcBackfillExecutor<S> {
         } else if self.options.disable_backfill {
             // If backfill is disabled, we just mark the backfill as finished
             tracing::info!(
-                upstream_table_id,
+                table_id,
                 upstream_table_name,
                 "CdcBackfill has been disabled"
             );
@@ -585,7 +630,7 @@ impl<S: StateStore> CdcBackfillExecutor<S> {
         drop(upstream_table_reader);
 
         tracing::info!(
-            upstream_table_id,
+            table_id,
             upstream_table_name,
             "CdcBackfill has already finished and will forward messages directly to the downstream"
         );
@@ -610,7 +655,7 @@ impl<S: StateStore> CdcBackfillExecutor<S> {
 
                     // mark progress as finished
                     if let Some(progress) = self.progress.as_mut() {
-                        progress.finish(barrier.epoch.curr, total_snapshot_row_count);
+                        progress.finish(barrier.epoch, total_snapshot_row_count);
                     }
                     yield msg;
                     // break after the state have been saved
diff --git a/src/stream/src/executor/backfill/no_shuffle_backfill.rs b/src/stream/src/executor/backfill/no_shuffle_backfill.rs
index e368086a97737..a65c5f1c4487b 100644
--- a/src/stream/src/executor/backfill/no_shuffle_backfill.rs
+++ b/src/stream/src/executor/backfill/no_shuffle_backfill.rs
@@ -417,7 +417,7 @@ where
                 snapshot_read_epoch = barrier.epoch.prev;
 
                 self.progress.update(
-                    barrier.epoch.curr,
+                    barrier.epoch,
                     snapshot_read_epoch,
                     total_snapshot_processed_rows,
                 );
@@ -540,7 +540,7 @@ where
                     // and backfill which just finished, we need to update mview tracker,
                     // it does not persist this information.
                     self.progress
-                        .finish(barrier.epoch.curr, total_snapshot_processed_rows);
+                        .finish(barrier.epoch, total_snapshot_processed_rows);
                     tracing::trace!(
                         epoch = ?barrier.epoch,
                         "Updated CreateMaterializedTracker"
diff --git a/src/stream/src/executor/backfill/utils.rs b/src/stream/src/executor/backfill/utils.rs
index f67442e723198..7ae488bb718a1 100644
--- a/src/stream/src/executor/backfill/utils.rs
+++ b/src/stream/src/executor/backfill/utils.rs
@@ -30,7 +30,7 @@ use governor::{Quota, RateLimiter};
 use risingwave_common::array::stream_record::Record;
 use risingwave_common::array::{Op, StreamChunk};
 use risingwave_common::bail;
-use risingwave_common::buffer::BitmapBuilder;
+use risingwave_common::bitmap::BitmapBuilder;
 use risingwave_common::hash::{VirtualNode, VnodeBitmapExt};
 use risingwave_common::row::{OwnedRow, Row, RowExt};
 use risingwave_common::types::{DataType, Datum};
diff --git a/src/stream/src/executor/barrier_align.rs b/src/stream/src/executor/barrier_align.rs
index f485615791945..a29bf91bb6660 100644
--- a/src/stream/src/executor/barrier_align.rs
+++ b/src/stream/src/executor/barrier_align.rs
@@ -121,7 +121,7 @@ pub async fn barrier_align(
                         Message::Barrier(barrier) => {
                             yield AlignedMessage::Barrier(barrier);
                             right_barrier_align_duration
-                                .observe(start_time.elapsed().as_secs_f64());
+                                .inc_by(start_time.elapsed().as_nanos() as u64);
                             break;
                         }
                     }
@@ -144,7 +144,8 @@ pub async fn barrier_align(
                         Message::Chunk(chunk) => yield AlignedMessage::Left(chunk),
                         Message::Barrier(barrier) => {
                             yield AlignedMessage::Barrier(barrier);
-                            left_barrier_align_duration.observe(start_time.elapsed().as_secs_f64());
+                            left_barrier_align_duration
+                                .inc_by(start_time.elapsed().as_nanos() as u64);
                             break;
                         }
                     }
diff --git a/src/stream/src/executor/batch_query.rs b/src/stream/src/executor/batch_query.rs
index d7c7f38d99504..34e32189bc5e5 100644
--- a/src/stream/src/executor/batch_query.rs
+++ b/src/stream/src/executor/batch_query.rs
@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use futures::TryStreamExt;
 use risingwave_common::array::Op;
 use risingwave_hummock_sdk::HummockReadEpoch;
 use risingwave_storage::store::PrefetchOptions;
@@ -52,6 +53,7 @@ where
                 PrefetchOptions::prefetch_for_large_range_scan(),
             )
             .await?;
+        let iter = iter.map_ok(|keyed_row| keyed_row.into_owned_row());
         pin_mut!(iter);
 
         while let Some(data_chunk) =
diff --git a/src/stream/src/executor/chain.rs b/src/stream/src/executor/chain.rs
index 96b9422a97b2c..6f198ff2b7e12 100644
--- a/src/stream/src/executor/chain.rs
+++ b/src/stream/src/executor/chain.rs
@@ -64,7 +64,7 @@ impl ChainExecutor {
         // If the barrier is a conf change of creating this mview, and the snapshot is not to be
         // consumed, we can finish the progress immediately.
         if barrier.is_newly_added(self.actor_id) && self.upstream_only {
-            self.progress.finish(barrier.epoch.curr, 0);
+            self.progress.finish(barrier.epoch, 0);
         }
 
         // The first barrier message should be propagated.
@@ -88,7 +88,7 @@ impl ChainExecutor {
         for msg in upstream {
             let msg = msg?;
             if to_consume_snapshot && let Message::Barrier(barrier) = &msg {
-                self.progress.finish(barrier.epoch.curr, 0);
+                self.progress.finish(barrier.epoch, 0);
             }
             yield msg;
         }
@@ -143,6 +143,7 @@ mod test {
                     added_actors: maplit::hashset! { actor_id },
                     splits: Default::default(),
                     pause: false,
+                    subscriptions_to_add: vec![],
                 }),
             )),
             Message::Chunk(StreamChunk::from_pretty("I\n + 3")),
diff --git a/src/stream/src/executor/dedup/append_only_dedup.rs b/src/stream/src/executor/dedup/append_only_dedup.rs
index 8455bd2643e5c..fa110b9f84f17 100644
--- a/src/stream/src/executor/dedup/append_only_dedup.rs
+++ b/src/stream/src/executor/dedup/append_only_dedup.rs
@@ -15,7 +15,7 @@
 use futures::stream;
 use itertools::Itertools;
 use risingwave_common::array::Op;
-use risingwave_common::buffer::BitmapBuilder;
+use risingwave_common::bitmap::BitmapBuilder;
 use risingwave_common::row::RowExt;
 
 use super::cache::DedupCache;
diff --git a/src/stream/src/executor/dispatch.rs b/src/stream/src/executor/dispatch.rs
index 9e6503c2d9a28..9f452dc1863b0 100644
--- a/src/stream/src/executor/dispatch.rs
+++ b/src/stream/src/executor/dispatch.rs
@@ -20,7 +20,7 @@ use std::ops::{Deref, DerefMut};
 use futures::TryStreamExt;
 use itertools::Itertools;
 use risingwave_common::array::Op;
-use risingwave_common::buffer::BitmapBuilder;
+use risingwave_common::bitmap::BitmapBuilder;
 use risingwave_common::hash::{ActorMapping, ExpandedActorMapping, VirtualNode};
 use risingwave_common::metrics::LabelGuardedIntCounter;
 use risingwave_common::util::iter_util::ZipEqFast;
@@ -31,7 +31,9 @@ use tokio::time::Instant;
 use tracing::{event, Instrument};
 
 use super::exchange::output::{new_output, BoxedOutput};
-use super::{AddMutation, TroublemakerExecutor, UpdateMutation};
+use super::{
+    AddMutation, DispatcherBarrier, DispatcherMessage, TroublemakerExecutor, UpdateMutation,
+};
 use crate::executor::prelude::*;
 use crate::executor::StreamConsumer;
 use crate::task::{DispatcherId, SharedContext};
@@ -142,7 +144,9 @@ impl DispatchExecutorInner {
                     .map(Ok)
                     .try_for_each_concurrent(limit, |dispatcher| async {
                         let start_time = Instant::now();
-                        dispatcher.dispatch_barrier(barrier.clone()).await?;
+                        dispatcher
+                            .dispatch_barrier(barrier.clone().into_dispatcher())
+                            .await?;
                         dispatcher
                             .actor_output_buffer_blocking_duration_ns
                             .inc_by(start_time.elapsed().as_nanos() as u64);
@@ -497,7 +501,7 @@ macro_rules! impl_dispatcher {
                 }
             }
 
-            pub async fn dispatch_barrier(&mut self, barrier: Barrier) -> StreamResult<()> {
+            pub async fn dispatch_barrier(&mut self, barrier: DispatcherBarrier) -> StreamResult<()> {
                 match self {
                     $( Self::$variant_name(inner) => inner.dispatch_barrier(barrier).await, )*
                 }
@@ -561,7 +565,7 @@ pub trait Dispatcher: Debug + 'static {
     /// Dispatch a data chunk to downstream actors.
     fn dispatch_data(&mut self, chunk: StreamChunk) -> impl DispatchFuture<'_>;
     /// Dispatch a barrier to downstream actors, generally by broadcasting it.
-    fn dispatch_barrier(&mut self, barrier: Barrier) -> impl DispatchFuture<'_>;
+    fn dispatch_barrier(&mut self, barrier: DispatcherBarrier) -> impl DispatchFuture<'_>;
     /// Dispatch a watermark to downstream actors, generally by broadcasting it.
     fn dispatch_watermark(&mut self, watermark: Watermark) -> impl DispatchFuture<'_>;
 
@@ -591,7 +595,7 @@ pub trait Dispatcher: Debug + 'static {
 /// always unlimited.
 async fn broadcast_concurrent(
     outputs: impl IntoIterator<Item = &'_ mut BoxedOutput>,
-    message: Message,
+    message: DispatcherMessage,
 ) -> StreamResult<()> {
     futures::future::try_join_all(
         outputs
@@ -637,21 +641,24 @@ impl Dispatcher for RoundRobinDataDispatcher {
             chunk.project(&self.output_indices)
         };
 
-        self.outputs[self.cur].send(Message::Chunk(chunk)).await?;
+        self.outputs[self.cur]
+            .send(DispatcherMessage::Chunk(chunk))
+            .await?;
         self.cur += 1;
         self.cur %= self.outputs.len();
         Ok(())
     }
 
-    async fn dispatch_barrier(&mut self, barrier: Barrier) -> StreamResult<()> {
+    async fn dispatch_barrier(&mut self, barrier: DispatcherBarrier) -> StreamResult<()> {
         // always broadcast barrier
-        broadcast_concurrent(&mut self.outputs, Message::Barrier(barrier)).await
+        broadcast_concurrent(&mut self.outputs, DispatcherMessage::Barrier(barrier)).await
     }
 
     async fn dispatch_watermark(&mut self, watermark: Watermark) -> StreamResult<()> {
         if let Some(watermark) = watermark.transform_with_indices(&self.output_indices) {
             // always broadcast watermark
-            broadcast_concurrent(&mut self.outputs, Message::Watermark(watermark)).await?;
+            broadcast_concurrent(&mut self.outputs, DispatcherMessage::Watermark(watermark))
+                .await?;
         }
         Ok(())
     }
@@ -725,15 +732,16 @@ impl Dispatcher for HashDataDispatcher {
         self.outputs.extend(outputs);
     }
 
-    async fn dispatch_barrier(&mut self, barrier: Barrier) -> StreamResult<()> {
+    async fn dispatch_barrier(&mut self, barrier: DispatcherBarrier) -> StreamResult<()> {
         // always broadcast barrier
-        broadcast_concurrent(&mut self.outputs, Message::Barrier(barrier)).await
+        broadcast_concurrent(&mut self.outputs, DispatcherMessage::Barrier(barrier)).await
     }
 
     async fn dispatch_watermark(&mut self, watermark: Watermark) -> StreamResult<()> {
         if let Some(watermark) = watermark.transform_with_indices(&self.output_indices) {
             // always broadcast watermark
-            broadcast_concurrent(&mut self.outputs, Message::Watermark(watermark)).await?;
+            broadcast_concurrent(&mut self.outputs, DispatcherMessage::Watermark(watermark))
+                .await?;
         }
         Ok(())
     }
@@ -818,7 +826,9 @@ impl Dispatcher for HashDataDispatcher {
                             "send = \n{:#?}",
                             new_stream_chunk
                         );
-                        output.send(Message::Chunk(new_stream_chunk)).await?;
+                        output
+                            .send(DispatcherMessage::Chunk(new_stream_chunk))
+                            .await?;
                     }
                     StreamResult::Ok(())
                 }),
@@ -888,18 +898,26 @@ impl Dispatcher for BroadcastDispatcher {
         } else {
             chunk.project(&self.output_indices)
         };
-        broadcast_concurrent(self.outputs.values_mut(), Message::Chunk(chunk)).await
+        broadcast_concurrent(self.outputs.values_mut(), DispatcherMessage::Chunk(chunk)).await
     }
 
-    async fn dispatch_barrier(&mut self, barrier: Barrier) -> StreamResult<()> {
+    async fn dispatch_barrier(&mut self, barrier: DispatcherBarrier) -> StreamResult<()> {
         // always broadcast barrier
-        broadcast_concurrent(self.outputs.values_mut(), Message::Barrier(barrier)).await
+        broadcast_concurrent(
+            self.outputs.values_mut(),
+            DispatcherMessage::Barrier(barrier),
+        )
+        .await
     }
 
     async fn dispatch_watermark(&mut self, watermark: Watermark) -> StreamResult<()> {
         if let Some(watermark) = watermark.transform_with_indices(&self.output_indices) {
             // always broadcast watermark
-            broadcast_concurrent(self.outputs.values_mut(), Message::Watermark(watermark)).await?;
+            broadcast_concurrent(
+                self.outputs.values_mut(),
+                DispatcherMessage::Watermark(watermark),
+            )
+            .await?;
         }
         Ok(())
     }
@@ -970,10 +988,12 @@ impl Dispatcher for SimpleDispatcher {
         assert!(self.output.len() <= 2);
     }
 
-    async fn dispatch_barrier(&mut self, barrier: Barrier) -> StreamResult<()> {
+    async fn dispatch_barrier(&mut self, barrier: DispatcherBarrier) -> StreamResult<()> {
         // Only barrier is allowed to be dispatched to multiple outputs during migration.
         for output in &mut self.output {
-            output.send(Message::Barrier(barrier.clone())).await?;
+            output
+                .send(DispatcherMessage::Barrier(barrier.clone()))
+                .await?;
         }
         Ok(())
     }
@@ -992,7 +1012,7 @@ impl Dispatcher for SimpleDispatcher {
         } else {
             chunk.project(&self.output_indices)
         };
-        output.send(Message::Chunk(chunk)).await
+        output.send(DispatcherMessage::Chunk(chunk)).await
     }
 
     async fn dispatch_watermark(&mut self, watermark: Watermark) -> StreamResult<()> {
@@ -1003,7 +1023,7 @@ impl Dispatcher for SimpleDispatcher {
             .expect("expect exactly one output");
 
         if let Some(watermark) = watermark.transform_with_indices(&self.output_indices) {
-            output.send(Message::Watermark(watermark)).await?;
+            output.send(DispatcherMessage::Watermark(watermark)).await?;
         }
         Ok(())
     }
@@ -1044,23 +1064,25 @@ mod tests {
     use crate::executor::exchange::output::Output;
     use crate::executor::exchange::permit::channel_for_test;
     use crate::executor::receiver::ReceiverExecutor;
+    use crate::executor::{BarrierInner as Barrier, MessageInner as Message};
+    use crate::task::barrier_test_utils::LocalBarrierTestEnv;
     use crate::task::test_utils::helper_make_local_actor;
 
     #[derive(Debug)]
     pub struct MockOutput {
         actor_id: ActorId,
-        data: Arc<Mutex<Vec<Message>>>,
+        data: Arc<Mutex<Vec<DispatcherMessage>>>,
     }
 
     impl MockOutput {
-        pub fn new(actor_id: ActorId, data: Arc<Mutex<Vec<Message>>>) -> Self {
+        pub fn new(actor_id: ActorId, data: Arc<Mutex<Vec<DispatcherMessage>>>) -> Self {
             Self { actor_id, data }
         }
     }
 
     #[async_trait]
     impl Output for MockOutput {
-        async fn send(&mut self, message: Message) -> StreamResult<()> {
+        async fn send(&mut self, message: DispatcherMessage) -> StreamResult<()> {
             self.data.lock().unwrap().push(message);
             Ok(())
         }
@@ -1154,7 +1176,11 @@ mod tests {
         let (tx, rx) = channel_for_test();
         let actor_id = 233;
         let fragment_id = 666;
-        let input = Executor::new(Default::default(), ReceiverExecutor::for_test(rx).boxed());
+        let barrier_test_env = LocalBarrierTestEnv::for_test().await;
+        let input = Executor::new(
+            Default::default(),
+            ReceiverExecutor::for_test(233, rx, barrier_test_env.shared_context.clone()).boxed(),
+        );
         let ctx = Arc::new(SharedContext::for_test());
         let metrics = Arc::new(StreamingMetrics::unused());
 
@@ -1245,7 +1271,10 @@ mod tests {
                 actor_new_dispatchers: Default::default(),
             },
         ));
-        tx.send(Message::Barrier(b1)).await.unwrap();
+        barrier_test_env.inject_barrier(&b1, [], [actor_id]);
+        tx.send(Message::Barrier(b1.clone().into_dispatcher()))
+            .await
+            .unwrap();
         executor.next().await.unwrap().unwrap();
 
         // 5. Check downstream.
@@ -1261,7 +1290,9 @@ mod tests {
         try_recv!(old_simple).unwrap().as_barrier().unwrap(); // Untouched.
 
         // 6. Send another barrier.
-        tx.send(Message::Barrier(Barrier::new_test_barrier(test_epoch(2))))
+        let b2 = Barrier::new_test_barrier(test_epoch(2));
+        barrier_test_env.inject_barrier(&b2, [], [actor_id]);
+        tx.send(Message::Barrier(b2.into_dispatcher()))
             .await
             .unwrap();
         executor.next().await.unwrap().unwrap();
@@ -1299,7 +1330,10 @@ mod tests {
                 actor_new_dispatchers: Default::default(),
             },
         ));
-        tx.send(Message::Barrier(b3)).await.unwrap();
+        barrier_test_env.inject_barrier(&b3, [], [actor_id]);
+        tx.send(Message::Barrier(b3.into_dispatcher()))
+            .await
+            .unwrap();
         executor.next().await.unwrap().unwrap();
 
         // 10. Check downstream.
@@ -1309,7 +1343,9 @@ mod tests {
         try_recv!(new_simple).unwrap().as_barrier().unwrap(); // Since it's just added, it won't receive the chunk.
 
         // 11. Send another barrier.
-        tx.send(Message::Barrier(Barrier::new_test_barrier(test_epoch(4))))
+        let b4 = Barrier::new_test_barrier(test_epoch(4));
+        barrier_test_env.inject_barrier(&b4, [], [actor_id]);
+        tx.send(Message::Barrier(b4.into_dispatcher()))
             .await
             .unwrap();
         executor.next().await.unwrap().unwrap();
@@ -1403,7 +1439,7 @@ mod tests {
             } else {
                 let message = guard.first().unwrap();
                 let real_chunk = match message {
-                    Message::Chunk(chunk) => chunk,
+                    DispatcherMessage::Chunk(chunk) => chunk,
                     _ => panic!(),
                 };
                 real_chunk
diff --git a/src/stream/src/executor/dynamic_filter.rs b/src/stream/src/executor/dynamic_filter.rs
index adba9aded3fa9..ff44399b22a34 100644
--- a/src/stream/src/executor/dynamic_filter.rs
+++ b/src/stream/src/executor/dynamic_filter.rs
@@ -16,8 +16,7 @@ use std::ops::Bound::{self, *};
 
 use futures::stream;
 use risingwave_common::array::{Array, ArrayImpl, Op};
-use risingwave_common::bail;
-use risingwave_common::buffer::{Bitmap, BitmapBuilder};
+use risingwave_common::bitmap::{Bitmap, BitmapBuilder};
 use risingwave_common::hash::VnodeBitmapExt;
 use risingwave_common::row::{self, once, OwnedRow as RowData};
 use risingwave_common::types::{DefaultOrd, ToDatumRef, ToOwnedDatum};
@@ -25,7 +24,7 @@ use risingwave_common::util::iter_util::ZipEqDebug;
 use risingwave_expr::expr::{
     build_func_non_strict, InputRefExpression, LiteralExpression, NonStrictExpression,
 };
-use risingwave_pb::expr::expr_node::Type as ExprNodeType;
+use risingwave_pb::expr::expr_node::Type as PbExprNodeType;
 use risingwave_pb::expr::expr_node::Type::{
     GreaterThan, GreaterThanOrEqual, LessThan, LessThanOrEqual,
 };
@@ -46,16 +45,12 @@ pub struct DynamicFilterExecutor<S: StateStore, const USE_WATERMARK_CACHE: bool>
     source_l: Option<Executor>,
     source_r: Option<Executor>,
     key_l: usize,
-    comparator: ExprNodeType,
+    comparator: PbExprNodeType,
     left_table: WatermarkCacheParameterizedStateTable<S, USE_WATERMARK_CACHE>,
     right_table: StateTable<S>,
     metrics: Arc<StreamingMetrics>,
     /// The maximum size of the chunk produced by executor at a time.
     chunk_size: usize,
-    /// If the right side's change always make the condition more relaxed.
-    /// In other words, make more record in the left side satisfy the condition.
-    /// In that case, there are only records which does not satisfy the condition in the table.
-    condition_always_relax: bool,
     cleaned_by_watermark: bool,
 }
 
@@ -68,12 +63,11 @@ impl<S: StateStore, const USE_WATERMARK_CACHE: bool> DynamicFilterExecutor<S, US
         source_l: Executor,
         source_r: Executor,
         key_l: usize,
-        comparator: ExprNodeType,
+        comparator: PbExprNodeType,
         state_table_l: WatermarkCacheParameterizedStateTable<S, USE_WATERMARK_CACHE>,
         state_table_r: StateTable<S>,
         metrics: Arc<StreamingMetrics>,
         chunk_size: usize,
-        condition_always_relax: bool,
         cleaned_by_watermark: bool,
     ) -> Self {
         Self {
@@ -88,7 +82,6 @@ impl<S: StateStore, const USE_WATERMARK_CACHE: bool> DynamicFilterExecutor<S, US
             right_table: state_table_r,
             metrics,
             chunk_size,
-            condition_always_relax,
             cleaned_by_watermark,
         }
     }
@@ -96,13 +89,20 @@ impl<S: StateStore, const USE_WATERMARK_CACHE: bool> DynamicFilterExecutor<S, US
     async fn apply_batch(
         &mut self,
         chunk: &StreamChunk,
-        condition: Option<NonStrictExpression>,
+        filter_condition: Option<NonStrictExpression>,
+        below_watermark_condition: Option<NonStrictExpression>,
     ) -> Result<(Vec<Op>, Bitmap), StreamExecutorError> {
         let mut new_ops = Vec::with_capacity(chunk.capacity());
         let mut new_visibility = BitmapBuilder::with_capacity(chunk.capacity());
         let mut last_res = false;
 
-        let eval_results = if let Some(cond) = condition {
+        let filter_results = if let Some(cond) = filter_condition {
+            Some(cond.eval_infallible(chunk).await)
+        } else {
+            None
+        };
+
+        let below_watermark = if let Some(cond) = below_watermark_condition {
             Some(cond.eval_infallible(chunk).await)
         } else {
             None
@@ -111,16 +111,26 @@ impl<S: StateStore, const USE_WATERMARK_CACHE: bool> DynamicFilterExecutor<S, US
         for (idx, (op, row)) in chunk.rows().enumerate() {
             let left_val = row.datum_at(self.key_l).to_owned_datum();
 
-            let satisfied_dyn_filter_cond = if let Some(array) = &eval_results {
+            let satisfied_dyn_filter_cond = if let Some(array) = &filter_results {
                 if let ArrayImpl::Bool(results) = &**array {
                     results.value_at(idx).unwrap_or(false)
                 } else {
-                    panic!("condition eval must return bool array")
+                    panic!("dynamic filter condition eval must return bool array")
                 }
             } else {
                 // A NULL right value implies a false evaluation for all rows
                 false
             };
+            let below_watermark = if let Some(array) = &below_watermark {
+                if let ArrayImpl::Bool(results) = &**array {
+                    results.value_at(idx).unwrap_or(false)
+                } else {
+                    panic!("below watermark check condition eval must return bool array")
+                }
+            } else {
+                // there was no state cleaning watermark before
+                false
+            };
 
             match op {
                 Op::Insert | Op::Delete => {
@@ -162,15 +172,28 @@ impl<S: StateStore, const USE_WATERMARK_CACHE: bool> DynamicFilterExecutor<S, US
                 },
             }
 
-            // Do not need to maintain the emitted records in the left table when the condition is always relaxed
-            // Also, if the delete value satisfy the condition, there could not be in the state table.
-            if self.condition_always_relax && satisfied_dyn_filter_cond {
+            // No need to maintain the records in the left table after the records become
+            // below state cleaning watermark.
+            // Here's why:
+            // 1. For >= and >, watermark on rhs means that any future changes that satisfy the
+            //    filter condition should >= state clean watermark, and those below the watermark
+            //    will never satisfy the filter condition.
+            // 2. For < and <=, watermark on rhs means that any future changes that are below
+            //    the watermark will always satisfy the filter condition, so those changes should
+            //    always be directly sent to downstream without any state table maintenance.
+            if below_watermark {
                 continue;
             }
+
             // Store the rows without a null left key
             // null key in left side of predicate should never be stored
             // (it will never satisfy the filter condition)
             if left_val.is_some() {
+                // Note that when updating `key_l` column, in most cases the timestamp column, if the
+                // timestamp is updated from a value below watermark to a value above watermark, it's
+                // possible that, due to state cleaning strategy, the old row may still exist in the
+                // state table. We have to be careful about the state table insertion. Now it works
+                // because the `key_l` column is in the state table pk as inferred by the frontend.
                 match op {
                     Op::Insert | Op::UpdateInsert => {
                         self.left_table.insert(row);
@@ -266,12 +289,15 @@ impl<S: StateStore, const USE_WATERMARK_CACHE: bool> DynamicFilterExecutor<S, US
         let r_data_type = input_r.schema().data_types()[0].clone();
         // The types are aligned by frontend.
         assert_eq!(l_data_type, r_data_type);
-        let dynamic_cond = {
+
+        let build_cond = {
+            let l_data_type = l_data_type.clone();
+            let r_data_type = r_data_type.clone();
             let eval_error_report = self.eval_error_report.clone();
-            move |literal: Datum| {
+            move |cmp: PbExprNodeType, literal: Datum| {
                 literal.map(|scalar| {
                     build_func_non_strict(
-                        self.comparator,
+                        cmp,
                         DataType::Boolean,
                         vec![
                             Box::new(InputRefExpression::new(l_data_type.clone(), self.key_l)),
@@ -298,14 +324,14 @@ impl<S: StateStore, const USE_WATERMARK_CACHE: bool> DynamicFilterExecutor<S, US
         self.right_table.init_epoch(barrier.epoch);
         self.left_table.init_epoch(barrier.epoch);
 
-        let recovered_row = self.recover_rhs().await?;
-        let recovered_value = recovered_row.as_ref().map(|r| r[0].clone());
-        // At the beginning of an epoch, the `prev_epoch_value` == `current_epoch_value`
-        let mut prev_epoch_value: Option<Datum> = recovered_value.clone();
-        let mut current_epoch_value: Option<Datum> = recovered_value;
+        let recovered_rhs = self.recover_rhs().await?;
+        let recovered_rhs_value = recovered_rhs.as_ref().map(|r| r[0].clone());
+        // At the beginning of an epoch, the `committed_rhs_value` == `staging_rhs_value`
+        let mut committed_rhs_value: Option<Datum> = recovered_rhs_value.clone();
+        let mut staging_rhs_value: Option<Datum> = recovered_rhs_value;
         // This is only required to be some if the row arrived during this epoch.
-        let mut current_epoch_row = recovered_row.clone();
-        let mut last_committed_epoch_row = recovered_row;
+        let mut committed_rhs_row = recovered_rhs.clone();
+        let mut staging_rhs_row = recovered_rhs;
 
         // The first barrier message should be propagated.
         yield Message::Barrier(barrier);
@@ -313,22 +339,35 @@ impl<S: StateStore, const USE_WATERMARK_CACHE: bool> DynamicFilterExecutor<S, US
         let mut stream_chunk_builder =
             StreamChunkBuilder::new(self.chunk_size, self.schema.data_types());
 
-        let watermark_can_clean_state = !matches!(self.comparator, LessThan | LessThanOrEqual);
-        let mut unused_clean_hint = None;
-        let mut buffered_right_watermark = None;
+        let mut staging_state_watermark = None;
+        let mut watermark_to_propagate = None;
+        let can_propagate_watermark = matches!(self.comparator, GreaterThan | GreaterThanOrEqual);
+
         #[for_await]
         for msg in aligned_stream {
             match msg? {
                 AlignedMessage::Left(chunk) => {
                     // Reuse the logic from `FilterExecutor`
                     let chunk = chunk.compact(); // Is this unnecessary work?
-                    let right_val = prev_epoch_value.clone().flatten();
 
                     // The condition is `None` if it is always false by virtue of a NULL right
-                    // input, so we save evaluating it on the datachunk
-                    let condition = dynamic_cond(right_val).transpose()?;
-
-                    let (new_ops, new_visibility) = self.apply_batch(&chunk, condition).await?;
+                    // input, so we save evaluating it on the datachunk.
+                    let filter_condition =
+                        build_cond(self.comparator, committed_rhs_value.clone().flatten())
+                            .transpose()?;
+
+                    // The condition is `None` if there's no committed state cleaning watermark before.
+                    // Note that we should not use `state_cleaning_watermark` variable here, because
+                    // it represents outstanding watermark to be applied. Here we need the watermark
+                    // that has been applied to the state table, just like why we use `prev_epoch_value`
+                    // instead of `current_epoch_value` for `filter_condition`.
+                    let below_watermark_condition =
+                        build_cond(LessThan, self.left_table.get_committed_watermark().cloned())
+                            .transpose()?;
+
+                    let (new_ops, new_visibility) = self
+                        .apply_batch(&chunk, filter_condition, below_watermark_condition)
+                        .await?;
 
                     let columns = chunk.into_parts().0.into_parts().0;
 
@@ -346,24 +385,24 @@ impl<S: StateStore, const USE_WATERMARK_CACHE: bool> DynamicFilterExecutor<S, US
                     for (row, op) in data_chunk.rows().zip_eq_debug(ops.iter()) {
                         match *op {
                             Op::UpdateInsert | Op::Insert => {
-                                current_epoch_value = Some(row.datum_at(0).to_owned_datum());
-                                current_epoch_row = Some(row.into_owned_row());
+                                staging_rhs_value = Some(row.datum_at(0).to_owned_datum());
+                                staging_rhs_row = Some(row.into_owned_row());
                             }
-                            _ => {
+                            Op::UpdateDelete | Op::Delete => {
                                 // To be consistent, there must be an existing `current_epoch_value`
                                 // equivalent to row indicated for
                                 // deletion.
                                 if Some(row.datum_at(0))
-                                    != current_epoch_value.as_ref().map(ToDatumRef::to_datum_ref)
+                                    != staging_rhs_value.as_ref().map(ToDatumRef::to_datum_ref)
                                 {
                                     consistency_panic!(
-                                        current = ?current_epoch_value,
+                                        current = ?staging_rhs_value,
                                         to_delete = ?row,
                                         "inconsistent delete",
                                     );
                                 }
-                                current_epoch_value = None;
-                                current_epoch_row = None;
+                                staging_rhs_value = None;
+                                staging_rhs_row = None;
                             }
                         }
                     }
@@ -372,32 +411,28 @@ impl<S: StateStore, const USE_WATERMARK_CACHE: bool> DynamicFilterExecutor<S, US
                     // Do nothing.
                 }
                 AlignedMessage::WatermarkRight(watermark) => {
-                    if watermark_can_clean_state {
-                        unused_clean_hint = Some(watermark.val.clone());
-                        buffered_right_watermark = Some(watermark);
+                    if self.cleaned_by_watermark {
+                        staging_state_watermark = Some(watermark.val.clone());
+                    }
+                    if can_propagate_watermark {
+                        watermark_to_propagate = Some(watermark);
                     }
                 }
                 AlignedMessage::Barrier(barrier) => {
-                    // Flush the difference between the `prev_value` and `current_value`
+                    // Commit the staging RHS value.
                     //
                     // This block is guaranteed to be idempotent even if we may encounter multiple
-                    // barriers since `prev_epoch_value` is always be reset to
-                    // the equivalent of `current_epoch_value` at the end of
-                    // this block. Likewise, `last_committed_epoch_row` will always be equal to
-                    // `current_epoch_row`.
-                    // It is thus guaranteed not to commit state or produce chunks as long as
-                    // no new chunks have arrived since the previous barrier.
-                    let curr: Datum = current_epoch_value.clone().flatten();
-                    let prev: Datum = prev_epoch_value.flatten();
+                    // barriers since `committed_rhs_value` is always be reset to the equivalent of
+                    // `staging_rhs_value` at the end of this block. Likewise, `committed_rhs_row`
+                    // will always be equal to `staging_rhs_row`. It is thus guaranteed not to commit
+                    // state or produce chunks as long as no new chunks have arrived since the previous
+                    // barrier.
+                    let curr: Datum = staging_rhs_value.clone().flatten();
+                    let prev: Datum = committed_rhs_value.flatten();
                     if prev != curr {
                         let (range, _latest_is_lower, is_insert) = self.get_range(&curr, prev);
-                        if !is_insert && self.condition_always_relax {
-                            bail!("The optimizer inferred that the right side's change always make the condition more relaxed.\
-                                But the right changes make the conditions stricter.");
-                        }
 
                         let range = (Self::to_row_bound(range.0), Self::to_row_bound(range.1));
-
                         // TODO: prefetching for append-only case.
                         let streams = futures::future::try_join_all(
                             self.left_table.vnodes().iter_vnodes().map(|vnode| {
@@ -415,9 +450,6 @@ impl<S: StateStore, const USE_WATERMARK_CACHE: bool> DynamicFilterExecutor<S, US
                         #[for_await]
                         for res in stream::select_all(streams) {
                             let row = res?;
-                            if self.condition_always_relax && !self.cleaned_by_watermark {
-                                unused_clean_hint.clone_from(&row[self.key_l])
-                            }
                             if let Some(chunk) = stream_chunk_builder.append_row(
                                 // All rows have a single identity at this point
                                 if is_insert { Op::Insert } else { Op::Delete },
@@ -432,39 +464,36 @@ impl<S: StateStore, const USE_WATERMARK_CACHE: bool> DynamicFilterExecutor<S, US
                         }
                     }
 
-                    if let Some(watermark) = unused_clean_hint.take() {
-                        self.left_table.update_watermark(watermark, false);
+                    if let Some(watermark) = staging_state_watermark.take() {
+                        self.left_table.update_watermark(watermark.clone());
                     };
 
-                    if let Some(mut watermark) = buffered_right_watermark.take() {
-                        watermark.col_idx = self.key_l;
-                        yield Message::Watermark(watermark);
+                    if let Some(watermark) = watermark_to_propagate.take() {
+                        yield Message::Watermark(watermark.with_idx(self.key_l));
                     };
 
                     // Update the committed value on RHS if it has changed.
-                    if last_committed_epoch_row != current_epoch_row {
+                    if committed_rhs_row != staging_rhs_row {
                         // Only write the RHS value if this actor is in charge of vnode 0 on LHS
                         // Otherwise, we only actively replicate the changes.
                         if self.left_table.vnodes().is_set(0) {
                             // If both `None`, then this branch is inactive.
                             // Hence, at least one is `Some`, hence at least one update.
-                            if let Some(old_row) = last_committed_epoch_row.take() {
+                            if let Some(old_row) = committed_rhs_row.take() {
                                 self.right_table.delete(old_row);
                             }
-                            if let Some(row) = &current_epoch_row {
+                            if let Some(row) = &staging_rhs_row {
                                 self.right_table.insert(row);
                             }
                         }
-                        // Update the last committed row since it has changed
-                        last_committed_epoch_row.clone_from(&current_epoch_row);
                     }
 
                     self.left_table.commit(barrier.epoch).await?;
                     self.right_table.commit(barrier.epoch).await?;
 
-                    prev_epoch_value = Some(curr);
-
-                    debug_assert_eq!(last_committed_epoch_row, current_epoch_row);
+                    // Update the last committed RHS row and value.
+                    committed_rhs_row.clone_from(&staging_rhs_row);
+                    committed_rhs_value = Some(curr);
 
                     // Update the vnode bitmap for the left state table if asked.
                     if let Some(vnode_bitmap) = barrier.as_update_vnode_bitmap(self.ctx.id) {
@@ -525,18 +554,11 @@ mod tests {
     }
 
     async fn create_executor(
-        comparator: ExprNodeType,
-    ) -> (MessageSender, MessageSender, BoxedMessageStream) {
-        let mem_state = MemoryStateStore::new();
-        create_executor_inner(comparator, mem_state, false).await
-    }
-
-    async fn create_executor_inner(
-        comparator: ExprNodeType,
-        mem_state: MemoryStateStore,
-        always_relax: bool,
+        comparator: PbExprNodeType,
+        store: MemoryStateStore,
+        cleaned_by_watermark: bool,
     ) -> (MessageSender, MessageSender, BoxedMessageStream) {
-        let (mem_state_l, mem_state_r) = create_in_memory_state_table(mem_state).await;
+        let (mem_state_l, mem_state_r) = create_in_memory_state_table(store).await;
         let schema = Schema {
             fields: vec![Field::unnamed(DataType::Int64)],
         };
@@ -562,8 +584,7 @@ mod tests {
             mem_state_r,
             Arc::new(StreamingMetrics::unused()),
             1024,
-            always_relax,
-            false,
+            cleaned_by_watermark,
         );
         (tx_l, tx_r, executor.boxed().execute())
     }
@@ -598,9 +619,9 @@ mod tests {
             "  I
              + 4",
         );
-        let mem_state = MemoryStateStore::new();
+        let mem_store = MemoryStateStore::new();
         let (mut tx_l, mut tx_r, mut dynamic_filter) =
-            create_executor_inner(ExprNodeType::GreaterThan, mem_state.clone(), false).await;
+            create_executor(PbExprNodeType::GreaterThan, mem_store.clone(), false).await;
 
         // push the init barrier for left and right
         tx_l.push_barrier(test_epoch(1), false);
@@ -623,7 +644,7 @@ mod tests {
 
         // Recover executor from state store
         let (mut tx_l, mut tx_r, mut dynamic_filter) =
-            create_executor_inner(ExprNodeType::GreaterThan, mem_state.clone(), false).await;
+            create_executor(PbExprNodeType::GreaterThan, mem_store.clone(), false).await;
 
         // push the recovery barrier for left and right
         tx_l.push_barrier(test_epoch(2), false);
@@ -670,7 +691,7 @@ mod tests {
 
         // Recover executor from state store
         let (mut tx_l, mut tx_r, mut dynamic_filter) =
-            create_executor_inner(ExprNodeType::GreaterThan, mem_state.clone(), false).await;
+            create_executor(PbExprNodeType::GreaterThan, mem_store.clone(), false).await;
 
         // push recovery barrier
         tx_l.push_barrier(test_epoch(3), false);
@@ -755,8 +776,9 @@ mod tests {
             "  I
              + 4",
         );
+        let mem_store = MemoryStateStore::new();
         let (mut tx_l, mut tx_r, mut dynamic_filter) =
-            create_executor(ExprNodeType::GreaterThan).await;
+            create_executor(PbExprNodeType::GreaterThan, mem_store, false).await;
 
         // push the init barrier for left and right
         tx_l.push_barrier(test_epoch(1), false);
@@ -861,8 +883,9 @@ mod tests {
             "  I
              + 5",
         );
+        let mem_store = MemoryStateStore::new();
         let (mut tx_l, mut tx_r, mut dynamic_filter) =
-            create_executor(ExprNodeType::GreaterThanOrEqual).await;
+            create_executor(PbExprNodeType::GreaterThanOrEqual, mem_store, false).await;
 
         // push the init barrier for left and right
         tx_l.push_barrier(test_epoch(1), false);
@@ -967,8 +990,9 @@ mod tests {
             "  I
              + 1",
         );
+        let mem_store = MemoryStateStore::new();
         let (mut tx_l, mut tx_r, mut dynamic_filter) =
-            create_executor(ExprNodeType::LessThan).await;
+            create_executor(PbExprNodeType::LessThan, mem_store, false).await;
 
         // push the init barrier for left and right
         tx_l.push_barrier(test_epoch(1), false);
@@ -1073,8 +1097,9 @@ mod tests {
             "  I
              + 0",
         );
+        let mem_store = MemoryStateStore::new();
         let (mut tx_l, mut tx_r, mut dynamic_filter) =
-            create_executor(ExprNodeType::LessThanOrEqual).await;
+            create_executor(PbExprNodeType::LessThanOrEqual, mem_store, false).await;
 
         // push the init barrier for left and right
         tx_l.push_barrier(test_epoch(1), false);
@@ -1166,9 +1191,10 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn test_dynamic_filter_always_relax() -> StreamExecutorResult<()> {
+    async fn test_dynamic_filter_state_cleaning() -> StreamExecutorResult<()> {
         let chunk_l1 = StreamChunk::from_pretty(
             "  I
+             + 1
              + 2
              + 3
              + 4
@@ -1184,17 +1210,19 @@ mod tests {
             "  I
              + 2",
         );
+        let watermark_r1 = 2;
         let chunk_r2 = StreamChunk::from_pretty(
             "  I
              + 5",
         );
+        let watermark_r2 = 5;
 
-        let mem_state = MemoryStateStore::new();
+        let mem_store = MemoryStateStore::new();
         let (mut tx_l, mut tx_r, mut dynamic_filter) =
-            create_executor_inner(ExprNodeType::LessThanOrEqual, mem_state.clone(), true).await;
+            create_executor(PbExprNodeType::LessThanOrEqual, mem_store.clone(), true).await;
         let column_descs = ColumnDesc::unnamed(ColumnId::new(0), DataType::Int64);
         let table = StorageTable::for_test(
-            mem_state.clone(),
+            mem_store.clone(),
             TableId::new(0),
             vec![column_descs],
             vec![OrderType::ascending()],
@@ -1207,69 +1235,48 @@ mod tests {
         tx_r.push_barrier(test_epoch(1), false);
         dynamic_filter.next_unwrap_ready_barrier()?;
 
-        // push the 1st right chunk
+        // push the 1st set of messages
+        tx_l.push_chunk(chunk_l1);
         tx_r.push_chunk(chunk_r1);
-
-        // push the init barrier for left and right
+        tx_r.push_int64_watermark(0, watermark_r1);
         tx_l.push_barrier(test_epoch(2), false);
         tx_r.push_barrier(test_epoch(2), false);
 
-        // Get the barrier
-        dynamic_filter.next_unwrap_ready_barrier()?;
-
-        // push the 1st left chunk
-        tx_l.push_chunk(chunk_l1);
-
-        let chunk = dynamic_filter.next_unwrap_ready_chunk()?.compact();
+        let chunk = dynamic_filter.expect_chunk().await;
         assert_eq!(
-            chunk,
+            chunk.compact(),
             StreamChunk::from_pretty(
                 " I
+                + 1
                 + 2"
             )
         );
 
-        // push the init barrier for left and right
-        tx_l.push_barrier(test_epoch(3), false);
-        tx_r.push_barrier(test_epoch(3), false);
-
-        // Get the barrier
-        dynamic_filter.next_unwrap_ready_barrier()?;
+        dynamic_filter.expect_barrier().await;
 
-        assert!(!in_table(&table, 2).await);
+        assert!(!in_table(&table, 1).await); // `1` should be cleaned because it's less than watermark
+        assert!(in_table(&table, 2).await);
         assert!(in_table(&table, 3).await);
         assert!(in_table(&table, 4).await);
 
-        // push the 2nd left chunk
+        // push the 2nd set of messages
         tx_l.push_chunk(chunk_l2);
-        let chunk = dynamic_filter.next_unwrap_ready_chunk()?.compact();
+        tx_r.push_chunk(chunk_r2);
+        tx_r.push_int64_watermark(0, watermark_r2);
+        tx_l.push_barrier(test_epoch(3), false);
+        tx_r.push_barrier(test_epoch(3), false);
+
+        let chunk = dynamic_filter.expect_chunk().await;
         assert_eq!(
-            chunk,
+            chunk.compact(),
             StreamChunk::from_pretty(
+                // the two rows are directly sent to the output cuz they satisfy the condition of previously committed rhs
                 " I
                 + 1
                 - 2"
             )
         );
-        // push the init barrier for left and right
-        tx_l.push_barrier(test_epoch(4), false);
-        tx_r.push_barrier(test_epoch(4), false);
-        // Get the barrier
-        dynamic_filter.next_unwrap_ready_barrier()?;
-
-        assert!(!in_table(&table, 2).await);
-        assert!(!in_table(&table, 2).await);
-        assert!(!in_table(&table, 3).await);
-        assert!(in_table(&table, 4).await);
-
-        // push the 2nd right chunk
-        tx_r.push_chunk(chunk_r2);
-
-        // push the init barrier for left and right
-        tx_l.push_barrier(test_epoch(5), false);
-        tx_r.push_barrier(test_epoch(5), false);
-
-        let chunk = dynamic_filter.next_unwrap_ready_chunk()?.compact();
+        let chunk = dynamic_filter.expect_chunk().await;
         assert_eq!(
             chunk,
             StreamChunk::from_pretty(
@@ -1279,17 +1286,13 @@ mod tests {
             )
         );
 
-        // Get the barrier
-        dynamic_filter.next_unwrap_ready_barrier()?;
-        tx_l.push_barrier(test_epoch(6), false);
-        tx_r.push_barrier(test_epoch(6), false);
-        // Get the barrier
-        dynamic_filter.next_unwrap_ready_barrier()?;
-        // This part test need change the `DefaultWatermarkBufferStrategy` to `super::watermark::WatermarkNoBuffer`
-        // clean is the Bound::Exclude
-        // TODO: https://github.com/risingwavelabs/risingwave/issues/14014
-        // assert!(!in_table(&table, 4).await);
-        // assert!(in_table(&table, 5).await);
+        dynamic_filter.expect_barrier().await;
+
+        assert!(!in_table(&table, 2).await);
+        assert!(!in_table(&table, 3).await);
+        assert!(!in_table(&table, 4).await);
+        assert!(in_table(&table, 5).await);
+
         Ok(())
     }
 }
diff --git a/src/stream/src/executor/error.rs b/src/stream/src/executor/error.rs
index 41b8198b646a3..fa625d8bb8cec 100644
--- a/src/stream/src/executor/error.rs
+++ b/src/stream/src/executor/error.rs
@@ -66,12 +66,8 @@ pub enum ErrorKind {
         BoxedError,
     ),
 
-    #[error("Sink error: {0}")]
-    SinkError(
-        #[from]
-        #[backtrace]
-        SinkError,
-    ),
+    #[error("Sink error: sink_id={1}, error: {0}")]
+    SinkError(SinkError, u32),
 
     #[error(transparent)]
     RpcError(
@@ -94,11 +90,7 @@ pub enum ErrorKind {
     AlignBarrier(Box<Barrier>, Box<Barrier>),
 
     #[error("Connector error: {0}")]
-    ConnectorError(
-        #[source]
-        #[backtrace]
-        BoxedError,
-    ),
+    ConnectorError(BoxedError),
 
     #[error(transparent)]
     DmlError(
@@ -152,6 +144,12 @@ impl From<String> for StreamExecutorError {
     }
 }
 
+impl From<(SinkError, u32)> for StreamExecutorError {
+    fn from((err, sink_id): (SinkError, u32)) -> Self {
+        ErrorKind::SinkError(err, sink_id).into()
+    }
+}
+
 impl StreamExecutorError {
     pub fn variant_name(&self) -> &str {
         self.0.inner().as_ref()
diff --git a/src/stream/src/executor/exchange/input.rs b/src/stream/src/executor/exchange/input.rs
index 4c5648ab1dd49..e00a0da45979a 100644
--- a/src/stream/src/executor/exchange/input.rs
+++ b/src/stream/src/executor/exchange/input.rs
@@ -15,17 +15,19 @@
 use std::pin::Pin;
 use std::task::{Context, Poll};
 
-use anyhow::Context as _;
+use anyhow::{anyhow, Context as _};
 use futures::pin_mut;
 use futures_async_stream::try_stream;
 use pin_project::pin_project;
 use risingwave_common::util::addr::{is_local_address, HostAddr};
 use risingwave_pb::task_service::{permits, GetStreamResponse};
 use risingwave_rpc_client::ComputeClientPool;
+use tokio::sync::mpsc;
 
 use super::error::ExchangeChannelClosed;
 use super::permit::Receiver;
 use crate::executor::prelude::*;
+use crate::executor::{DispatcherBarrier, DispatcherMessage};
 use crate::task::{
     FragmentId, LocalBarrierManager, SharedContext, UpDownActorIds, UpDownFragmentIds,
 };
@@ -64,23 +66,80 @@ pub struct LocalInput {
 }
 type LocalInputStreamInner = impl MessageStream;
 
+async fn process_msg<'a>(
+    msg: DispatcherMessage,
+    get_mutation_subscriber: impl for<'b> FnOnce(
+            &'b DispatcherBarrier,
+        )
+            -> &'a mut mpsc::UnboundedReceiver<crate::task::SubscribeMutationItem>
+        + 'a,
+) -> StreamExecutorResult<Message> {
+    let barrier = match msg {
+        DispatcherMessage::Chunk(c) => {
+            return Ok(Message::Chunk(c));
+        }
+        DispatcherMessage::Barrier(b) => b,
+        DispatcherMessage::Watermark(watermark) => {
+            return Ok(Message::Watermark(watermark));
+        }
+    };
+    let mutation_subscriber = get_mutation_subscriber(&barrier);
+
+    let mutation = mutation_subscriber
+        .recv()
+        .await
+        .ok_or_else(|| anyhow!("failed to receive mutation of barrier {:?}", barrier))
+        .map(|(prev_epoch, mutation)| {
+            assert_eq!(prev_epoch, barrier.epoch.prev);
+            mutation
+        })?;
+    Ok(Message::Barrier(Barrier {
+        epoch: barrier.epoch,
+        mutation,
+        kind: barrier.kind,
+        tracing_context: barrier.tracing_context,
+        passed_actors: barrier.passed_actors,
+    }))
+}
+
 impl LocalInput {
-    pub fn new(channel: Receiver, actor_id: ActorId) -> Self {
+    pub fn new(
+        channel: Receiver,
+        upstream_actor_id: ActorId,
+        self_actor_id: ActorId,
+        local_barrier_manager: LocalBarrierManager,
+    ) -> Self {
         Self {
-            inner: Self::run(channel, actor_id),
-            actor_id,
+            inner: Self::run(
+                channel,
+                upstream_actor_id,
+                self_actor_id,
+                local_barrier_manager,
+            ),
+            actor_id: upstream_actor_id,
         }
     }
 
     #[try_stream(ok = Message, error = StreamExecutorError)]
-    async fn run(mut channel: Receiver, actor_id: ActorId) {
-        let span: await_tree::Span = format!("LocalInput (actor {actor_id})").into();
+    async fn run(
+        mut channel: Receiver,
+        upstream_actor_id: ActorId,
+        self_actor_id: ActorId,
+        local_barrier_manager: LocalBarrierManager,
+    ) {
+        let span: await_tree::Span = format!("LocalInput (actor {upstream_actor_id})").into();
+        let mut mutation_subscriber = None;
         while let Some(msg) = channel.recv().verbose_instrument_await(span.clone()).await {
-            yield msg;
+            yield process_msg(msg, |barrier| {
+                mutation_subscriber.get_or_insert_with(|| {
+                    local_barrier_manager.subscribe_barrier_mutation(self_actor_id, barrier)
+                })
+            })
+            .await?;
         }
         // Always emit an error outside the loop. This is because we use barrier as the control
         // message to stop the stream. Reaching here means the channel is closed unexpectedly.
-        Err(ExchangeChannelClosed::local_input(actor_id))?
+        Err(ExchangeChannelClosed::local_input(upstream_actor_id))?
     }
 }
 
@@ -147,6 +206,7 @@ impl RemoteInput {
         metrics: Arc<StreamingMetrics>,
         batched_permits_limit: usize,
     ) {
+        let self_actor_id = up_down_ids.1;
         let client = client_pool.get_by_addr(upstream_addr).await?;
         let (stream, permits_tx) = client
             .get_stream(up_down_ids.0, up_down_ids.1, up_down_frag.0, up_down_frag.1)
@@ -162,17 +222,18 @@ impl RemoteInput {
         let span: await_tree::Span = format!("RemoteInput (actor {up_actor_id})").into();
 
         let mut batched_permits_accumulated = 0;
+        let mut mutation_subscriber = None;
 
         pin_mut!(stream);
         while let Some(data_res) = stream.next().verbose_instrument_await(span.clone()).await {
             match data_res {
                 Ok(GetStreamResponse { message, permits }) => {
                     let msg = message.unwrap();
-                    let bytes = Message::get_encoded_len(&msg);
+                    let bytes = DispatcherMessage::get_encoded_len(&msg);
 
                     exchange_frag_recv_size_metrics.inc_by(bytes as u64);
 
-                    let msg_res = Message::from_protobuf(&msg);
+                    let msg_res = DispatcherMessage::from_protobuf(&msg);
                     if let Some(add_back_permits) = match permits.unwrap().value {
                         // For records, batch the permits we received to reduce the backward
                         // `AddPermits` messages.
@@ -194,23 +255,14 @@ impl RemoteInput {
                             .context("RemoteInput backward permits channel closed.")?;
                     }
 
-                    let mut msg = msg_res.context("RemoteInput decode message error")?;
-
-                    // Read barrier mutation from local barrier manager and attach it to the barrier message.
-                    if cfg!(not(test)) {
-                        if let Message::Barrier(barrier) = &mut msg {
-                            assert!(
-                                barrier.mutation.is_none(),
-                                "Mutation should be erased in remote side"
-                            );
-                            let mutation = local_barrier_manager
-                                .read_barrier_mutation(barrier)
-                                .await
-                                .context("Read barrier mutation error")?;
-                            barrier.mutation = mutation;
-                        }
-                    }
-                    yield msg;
+                    let msg = msg_res.context("RemoteInput decode message error")?;
+
+                    yield process_msg(msg, |barrier| {
+                        mutation_subscriber.get_or_insert_with(|| {
+                            local_barrier_manager.subscribe_barrier_mutation(self_actor_id, barrier)
+                        })
+                    })
+                    .await?;
                 }
 
                 Err(e) => Err(ExchangeChannelClosed::remote_input(up_down_ids.0, Some(e)))?,
@@ -256,12 +308,14 @@ pub(crate) fn new_input(
         LocalInput::new(
             context.take_receiver((upstream_actor_id, actor_id))?,
             upstream_actor_id,
+            actor_id,
+            context.local_barrier_manager.clone(),
         )
         .boxed_input()
     } else {
         RemoteInput::new(
             context.local_barrier_manager.clone(),
-            context.compute_client_pool.clone(),
+            context.compute_client_pool.as_ref().to_owned(),
             upstream_addr,
             (upstream_actor_id, actor_id),
             (upstream_fragment_id, fragment_id),
diff --git a/src/stream/src/executor/exchange/output.rs b/src/stream/src/executor/exchange/output.rs
index 41b4b5b844759..145286f561e17 100644
--- a/src/stream/src/executor/exchange/output.rs
+++ b/src/stream/src/executor/exchange/output.rs
@@ -22,7 +22,7 @@ use risingwave_common::util::addr::is_local_address;
 use super::error::ExchangeChannelClosed;
 use super::permit::Sender;
 use crate::error::StreamResult;
-use crate::executor::Message;
+use crate::executor::DispatcherMessage as Message;
 use crate::task::{ActorId, SharedContext};
 
 /// `Output` provides an interface for `Dispatcher` to send data into downstream actors.
diff --git a/src/stream/src/executor/exchange/permit.rs b/src/stream/src/executor/exchange/permit.rs
index 159494355cff2..8c86eb2753811 100644
--- a/src/stream/src/executor/exchange/permit.rs
+++ b/src/stream/src/executor/exchange/permit.rs
@@ -19,7 +19,7 @@ use std::sync::Arc;
 use risingwave_pb::task_service::permits;
 use tokio::sync::{mpsc, AcquireError, Semaphore, SemaphorePermit};
 
-use crate::executor::Message;
+use crate::executor::DispatcherMessage as Message;
 
 /// Message with its required permits.
 ///
@@ -214,7 +214,7 @@ mod tests {
     use futures::FutureExt;
 
     use super::*;
-    use crate::executor::Barrier;
+    use crate::executor::DispatcherBarrier as Barrier;
 
     #[test]
     fn test_channel_close() {
diff --git a/src/stream/src/executor/filter.rs b/src/stream/src/executor/filter.rs
index b183719a9a677..53fb2dc6f1387 100644
--- a/src/stream/src/executor/filter.rs
+++ b/src/stream/src/executor/filter.rs
@@ -13,7 +13,7 @@
 // limitations under the License.
 
 use risingwave_common::array::{Array, ArrayImpl, Op};
-use risingwave_common::buffer::BitmapBuilder;
+use risingwave_common::bitmap::BitmapBuilder;
 use risingwave_common::util::iter_util::ZipEqFast;
 use risingwave_expr::expr::NonStrictExpression;
 
diff --git a/src/stream/src/executor/hash_agg.rs b/src/stream/src/executor/hash_agg.rs
index 2659e2c154c72..6c6a99f4c2421 100644
--- a/src/stream/src/executor/hash_agg.rs
+++ b/src/stream/src/executor/hash_agg.rs
@@ -17,7 +17,7 @@ use std::marker::PhantomData;
 
 use futures::stream;
 use itertools::Itertools;
-use risingwave_common::buffer::{Bitmap, BitmapBuilder};
+use risingwave_common::bitmap::{Bitmap, BitmapBuilder};
 use risingwave_common::hash::{HashKey, PrecomputedBuildHasher};
 use risingwave_common::util::epoch::EpochPair;
 use risingwave_common::util::iter_util::ZipEqFast;
@@ -489,7 +489,7 @@ impl<K: HashKey, S: StateStore> HashAggExecutor<K, S> {
         if let Some(watermark) = window_watermark {
             // Update watermark of state tables, for state cleaning.
             this.all_state_tables_mut()
-                .for_each(|table| table.update_watermark(watermark.clone(), false));
+                .for_each(|table| table.update_watermark(watermark.clone()));
         }
 
         // Flush distinct dedup state.
diff --git a/src/stream/src/executor/hash_join.rs b/src/stream/src/executor/hash_join.rs
index 0bf825138d92a..d12117d74e0ab 100644
--- a/src/stream/src/executor/hash_join.rs
+++ b/src/stream/src/executor/hash_join.rs
@@ -12,16 +12,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 use std::collections::{BTreeMap, HashSet};
-use std::num::NonZeroU32;
-use std::sync::LazyLock;
 use std::time::Duration;
 
-use governor::{Quota, RateLimiter};
 use itertools::Itertools;
 use multimap::MultiMap;
 use risingwave_common::array::Op;
 use risingwave_common::hash::{HashKey, NullBitmap};
-use risingwave_common::log::LogSuppresser;
 use risingwave_common::types::{DefaultOrd, ToOwnedDatum};
 use risingwave_common::util::epoch::EpochPair;
 use risingwave_common::util::iter_util::ZipEqDebug;
@@ -269,8 +265,10 @@ impl<K: HashKey, S: StateStore, const T: JoinTypePrimitive> HashJoinExecutor<K,
             .collect_vec();
 
         // If pk is contained in join key.
-        let pk_contained_in_jk_l = is_subset(state_pk_indices_l, state_join_key_indices_l.clone());
-        let pk_contained_in_jk_r = is_subset(state_pk_indices_r, state_join_key_indices_r.clone());
+        let pk_contained_in_jk_l =
+            is_subset(state_pk_indices_l.clone(), state_join_key_indices_l.clone());
+        let pk_contained_in_jk_r =
+            is_subset(state_pk_indices_r.clone(), state_join_key_indices_r.clone());
 
         // check whether join key contains pk in both side
         let append_only_optimize = is_append_only && pk_contained_in_jk_l && pk_contained_in_jk_r;
@@ -838,25 +836,17 @@ impl<K: HashKey, S: StateStore, const T: JoinTypePrimitive> HashJoinExecutor<K,
             if let Some(rows) = &matched_rows {
                 join_matched_join_keys.observe(rows.len() as _);
                 if rows.len() > high_join_amplification_threshold {
-                    static LOG_SUPPERSSER: LazyLock<LogSuppresser> = LazyLock::new(|| {
-                        LogSuppresser::new(RateLimiter::direct(Quota::per_minute(
-                            NonZeroU32::new(1).unwrap(),
-                        )))
-                    });
-                    if let Ok(suppressed_count) = LOG_SUPPERSSER.check() {
-                        let join_key_data_types = side_update.ht.join_key_data_types();
-                        let key = key.deserialize(join_key_data_types)?;
-                        tracing::warn!(target: "high_join_amplification",
-                            suppressed_count,
-                            matched_rows_len = rows.len(),
-                            update_table_id = side_update.ht.table_id(),
-                            match_table_id = side_match.ht.table_id(),
-                            join_key = ?key,
-                            actor_id = ctx.id,
-                            fragment_id = ctx.fragment_id,
-                            "large rows matched for join key"
-                        );
-                    }
+                    let join_key_data_types = side_update.ht.join_key_data_types();
+                    let key = key.deserialize(join_key_data_types)?;
+                    tracing::warn!(target: "high_join_amplification",
+                        matched_rows_len = rows.len(),
+                        update_table_id = side_update.ht.table_id(),
+                        match_table_id = side_match.ht.table_id(),
+                        join_key = ?key,
+                        actor_id = ctx.id,
+                        fragment_id = ctx.fragment_id,
+                        "large rows matched for join key"
+                    );
                 }
             } else {
                 join_matched_join_keys.observe(0.0)
@@ -3001,8 +2991,8 @@ mod tests {
             chunk,
             StreamChunk::from_pretty(
                 "  I I I I
-                U- 2 5 . .
-                U+ 2 5 2 7
+                U-  2 5 . .
+                U+  2 5 2 7
                 +  . . 4 8
                 +  . . 6 9"
             )
@@ -3023,6 +3013,67 @@ mod tests {
         Ok(())
     }
 
+    #[tokio::test]
+    async fn test_streaming_hash_full_outer_join_update() -> StreamExecutorResult<()> {
+        let (mut tx_l, mut tx_r, mut hash_join) =
+            create_classical_executor::<{ JoinType::FullOuter }>(false, false, None).await;
+
+        // push the init barrier for left and right
+        tx_l.push_barrier(test_epoch(1), false);
+        tx_r.push_barrier(test_epoch(1), false);
+        hash_join.next_unwrap_ready_barrier()?;
+
+        tx_l.push_chunk(StreamChunk::from_pretty(
+            "  I I
+             + 1 1
+            ",
+        ));
+        let chunk = hash_join.next_unwrap_ready_chunk()?;
+        assert_eq!(
+            chunk,
+            StreamChunk::from_pretty(
+                " I I I I
+                + 1 1 . ."
+            )
+        );
+
+        tx_r.push_chunk(StreamChunk::from_pretty(
+            "  I I
+             + 1 1
+            ",
+        ));
+        let chunk = hash_join.next_unwrap_ready_chunk()?;
+
+        assert_eq!(
+            chunk,
+            StreamChunk::from_pretty(
+                " I I I I
+                U- 1 1 . .
+                U+ 1 1 1 1"
+            )
+        );
+
+        tx_l.push_chunk(StreamChunk::from_pretty(
+            "   I I
+              - 1 1
+              + 1 2
+            ",
+        ));
+        let chunk = hash_join.next_unwrap_ready_chunk()?;
+        let chunk = chunk.compact();
+        assert_eq!(
+            chunk,
+            StreamChunk::from_pretty(
+                " I I I I
+                - 1 1 1 1
+                + 1 2 1 1
+                "
+            )
+        );
+
+        Ok(())
+    }
+
     #[tokio::test]
     async fn test_streaming_hash_full_outer_join_with_nonequi_condition() -> StreamExecutorResult<()>
     {
@@ -3093,8 +3144,8 @@ mod tests {
             chunk,
             StreamChunk::from_pretty(
                 "  I I I I
-                U- 2 5 . .
-                U+ 2 5 2 6
+                U-  2 5 . .
+                U+  2 5 2 6
                 +  . . 4 8
                 +  . . 3 4" /* regression test (#2420): 3 4 should be forwarded only once
                              * despite matching on eq join on 2
diff --git a/src/stream/src/executor/integration_tests.rs b/src/stream/src/executor/integration_tests.rs
index 8f73fe26ee7de..0b7415adac38b 100644
--- a/src/stream/src/executor/integration_tests.rs
+++ b/src/stream/src/executor/integration_tests.rs
@@ -34,7 +34,8 @@ use crate::executor::monitor::StreamingMetrics;
 use crate::executor::test_utils::agg_executor::{
     generate_agg_schema, new_boxed_simple_agg_executor,
 };
-use crate::task::{LocalBarrierManager, SharedContext};
+use crate::executor::{BarrierInner as Barrier, MessageInner as Message};
+use crate::task::barrier_test_utils::LocalBarrierTestEnv;
 
 /// This test creates a merger-dispatcher pair, and run a sum. Each chunk
 /// has 0~9 elements. We first insert the 10 chunks, then delete them,
@@ -45,9 +46,19 @@ async fn test_merger_sum_aggr() {
         time_zone: String::from("UTC"),
     };
 
-    let actor_ctx = ActorContext::for_test(0);
+    let barrier_test_env = LocalBarrierTestEnv::for_test().await;
+    let mut next_actor_id = 0;
+    let next_actor_id = &mut next_actor_id;
+    let mut actors = HashSet::new();
+    let mut gen_next_actor_id = || {
+        *next_actor_id += 1;
+        actors.insert(*next_actor_id);
+        *next_actor_id
+    };
     // `make_actor` build an actor to do local aggregation
-    let make_actor = |input_rx| {
+    let mut make_actor = |input_rx| {
+        let actor_id = gen_next_actor_id();
+        let actor_ctx = ActorContext::for_test(actor_id);
         let input_schema = Schema {
             fields: vec![Field::unnamed(DataType::Int64)],
         };
@@ -57,7 +68,8 @@ async fn test_merger_sum_aggr() {
                 pk_indices: PkIndices::new(),
                 identity: "ReceiverExecutor".to_string(),
             },
-            ReceiverExecutor::for_test(input_rx).boxed(),
+            ReceiverExecutor::for_test(actor_id, input_rx, barrier_test_env.shared_context.clone())
+                .boxed(),
         );
         let agg_calls = vec![
             AggCall::from_pretty("(count:int8)"),
@@ -72,13 +84,17 @@ async fn test_merger_sum_aggr() {
             input: aggregator.boxed(),
             channel: Box::new(LocalOutput::new(233, tx)),
         };
+
         let actor = Actor::new(
             consumer,
             vec![],
             StreamingMetrics::unused().into(),
-            actor_ctx.clone(),
+            actor_ctx,
             expr_context.clone(),
-            LocalBarrierManager::for_test(),
+            barrier_test_env
+                .shared_context
+                .local_barrier_manager
+                .clone(),
         );
         (actor, rx)
     };
@@ -90,7 +106,6 @@ async fn test_merger_sum_aggr() {
     let mut inputs = vec![];
     let mut outputs = vec![];
 
-    let ctx = Arc::new(SharedContext::for_test());
     let metrics = Arc::new(StreamingMetrics::unused());
 
     // create 17 local aggregation actors
@@ -103,6 +118,8 @@ async fn test_merger_sum_aggr() {
     }
 
     // create a round robin dispatcher, which dispatches messages to the actors
+
+    let actor_id = gen_next_actor_id();
     let (input, rx) = channel_for_test();
     let receiver_op = Executor::new(
         ExecutorInfo {
@@ -111,7 +128,7 @@ async fn test_merger_sum_aggr() {
             pk_indices: PkIndices::new(),
             identity: "ReceiverExecutor".to_string(),
         },
-        ReceiverExecutor::for_test(rx).boxed(),
+        ReceiverExecutor::for_test(actor_id, rx, barrier_test_env.shared_context.clone()).boxed(),
     );
     let dispatcher = DispatchExecutor::new(
         receiver_op,
@@ -122,7 +139,7 @@ async fn test_merger_sum_aggr() {
         ))],
         0,
         0,
-        ctx,
+        barrier_test_env.shared_context.clone(),
         metrics,
         config::default::developer::stream_chunk_size(),
     );
@@ -130,12 +147,17 @@ async fn test_merger_sum_aggr() {
         dispatcher,
         vec![],
         StreamingMetrics::unused().into(),
-        actor_ctx.clone(),
+        ActorContext::for_test(actor_id),
         expr_context.clone(),
-        LocalBarrierManager::for_test(),
+        barrier_test_env
+            .shared_context
+            .local_barrier_manager
+            .clone(),
     );
     handles.push(tokio::spawn(actor.run()));
 
+    let actor_ctx = ActorContext::for_test(gen_next_actor_id());
+
     // use a merge operator to collect data from dispatchers before sending them to aggregator
     let merger = Executor::new(
         ExecutorInfo {
@@ -147,7 +169,12 @@ async fn test_merger_sum_aggr() {
             pk_indices: PkIndices::new(),
             identity: "MergeExecutor".to_string(),
         },
-        MergeExecutor::for_test(outputs).boxed(),
+        MergeExecutor::for_test(
+            actor_ctx.id,
+            outputs,
+            barrier_test_env.shared_context.clone(),
+        )
+        .boxed(),
     );
 
     // for global aggregator, we need to sum data and sum row count
@@ -192,13 +219,18 @@ async fn test_merger_sum_aggr() {
         StreamingMetrics::unused().into(),
         actor_ctx.clone(),
         expr_context.clone(),
-        LocalBarrierManager::for_test(),
+        barrier_test_env
+            .shared_context
+            .local_barrier_manager
+            .clone(),
     );
     handles.push(tokio::spawn(actor.run()));
 
     let mut epoch = test_epoch(1);
+    let b1 = Barrier::new_test_barrier(epoch);
+    barrier_test_env.inject_barrier(&b1, [], actors.clone());
     input
-        .send(Message::Barrier(Barrier::new_test_barrier(epoch)))
+        .send(Message::Barrier(b1.into_dispatcher()))
         .await
         .unwrap();
     epoch.inc_epoch();
@@ -211,17 +243,19 @@ async fn test_merger_sum_aggr() {
             );
             input.send(Message::Chunk(chunk)).await.unwrap();
         }
+        let b = Barrier::new_test_barrier(epoch);
+        barrier_test_env.inject_barrier(&b, [], actors.clone());
         input
-            .send(Message::Barrier(Barrier::new_test_barrier(epoch)))
+            .send(Message::Barrier(b.into_dispatcher()))
             .await
             .unwrap();
         epoch.inc_epoch();
     }
+    let b = Barrier::new_test_barrier(epoch)
+        .with_mutation(Mutation::Stop(actors.clone().into_iter().collect()));
+    barrier_test_env.inject_barrier(&b, [], actors);
     input
-        .send(Message::Barrier(
-            Barrier::new_test_barrier(epoch)
-                .with_mutation(Mutation::Stop([0].into_iter().collect())),
-        ))
+        .send(Message::Barrier(b.into_dispatcher()))
         .await
         .unwrap();
 
@@ -241,7 +275,7 @@ struct MockConsumer {
 }
 
 impl StreamConsumer for MockConsumer {
-    type BarrierStream = impl Stream<Item = StreamResult<Barrier>> + Send;
+    type BarrierStream = impl Stream<Item = StreamResult<crate::executor::Barrier>> + Send;
 
     fn execute(self: Box<Self>) -> Self::BarrierStream {
         let mut input = self.input.execute();
@@ -268,7 +302,7 @@ pub struct SenderConsumer {
 }
 
 impl StreamConsumer for SenderConsumer {
-    type BarrierStream = impl Stream<Item = StreamResult<Barrier>> + Send;
+    type BarrierStream = impl Stream<Item = StreamResult<crate::executor::Barrier>> + Send;
 
     fn execute(self: Box<Self>) -> Self::BarrierStream {
         let mut input = self.input.execute();
@@ -279,7 +313,16 @@ impl StreamConsumer for SenderConsumer {
                 let msg = item?;
                 let barrier = msg.as_barrier().cloned();
 
-                channel.send(msg).await.expect("failed to send message");
+                channel
+                    .send(match msg {
+                        Message::Chunk(chunk) => DispatcherMessage::Chunk(chunk),
+                        Message::Barrier(barrier) => {
+                            DispatcherMessage::Barrier(barrier.into_dispatcher())
+                        }
+                        Message::Watermark(watermark) => DispatcherMessage::Watermark(watermark),
+                    })
+                    .await
+                    .expect("failed to send message");
 
                 if let Some(barrier) = barrier {
                     yield barrier;
diff --git a/src/stream/src/executor/join/builder.rs b/src/stream/src/executor/join/builder.rs
index 72208aa45ded8..c39e385fdee96 100644
--- a/src/stream/src/executor/join/builder.rs
+++ b/src/stream/src/executor/join/builder.rs
@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use risingwave_common::array::stream_chunk::StreamChunkMut;
 use risingwave_common::array::stream_chunk_builder::StreamChunkBuilder;
 use risingwave_common::array::{Op, RowRef, StreamChunk};
 use risingwave_common::row::{OwnedRow, Row};
@@ -156,6 +157,48 @@ impl<const T: JoinTypePrimitive, const SIDE: SideTypePrimitive> JoinChunkBuilder
         }
     }
 
+    pub fn post_process(c: StreamChunk) -> StreamChunk {
+        let mut c = StreamChunkMut::from(c);
+
+        // NOTE(st1page): remove the pattern `UpdateDel(k, old), UpdateIns(k, NULL), UpdateDel(k, NULL),  UpdateIns(k, new)`
+        // to avoid this issue <https://github.com/risingwavelabs/risingwave/issues/17450>
+        let mut i = 2;
+        while i < c.capacity() {
+            if c.op(i - 1) == Op::UpdateInsert
+                && c.op(i) == Op::UpdateDelete
+                && c.row_ref(i) == c.row_ref(i - 1)
+            {
+                if c.op(i - 2) == Op::UpdateDelete && c.op(i + 1) == Op::UpdateInsert {
+                    c.set_op(i - 2, Op::Delete);
+                    c.set_vis(i - 1, false);
+                    c.set_vis(i, false);
+                    c.set_op(i + 1, Op::Insert);
+                    i += 3;
+                } else {
+                    debug_assert!(
+                        false,
+                        "unexpected Op sequences {:?}, {:?}, {:?}, {:?}",
+                        c.op(i - 2),
+                        c.op(i - 1),
+                        c.op(i),
+                        c.op(i + 1)
+                    );
+                    warn!(
+                        "unexpected Op sequences {:?}, {:?}, {:?}, {:?}",
+                        c.op(i - 2),
+                        c.op(i - 1),
+                        c.op(i),
+                        c.op(i + 1)
+                    );
+                    i += 1;
+                }
+            } else {
+                i += 1;
+            }
+        }
+        c.into()
+    }
+
     pub fn with_match_on_insert(
         &mut self,
         row: &RowRef<'_>,
@@ -166,6 +209,7 @@ impl<const T: JoinTypePrimitive, const SIDE: SideTypePrimitive> JoinChunkBuilder
             if matched_row.is_zero_degree() && only_forward_matched_side(T, SIDE) {
                 self.stream_chunk_builder
                     .append_row_matched(Op::Delete, &matched_row.row)
+                    .map(Self::post_process)
             } else {
                 None
             }
@@ -174,6 +218,7 @@ impl<const T: JoinTypePrimitive, const SIDE: SideTypePrimitive> JoinChunkBuilder
             if matched_row.is_zero_degree() && only_forward_matched_side(T, SIDE) {
                 self.stream_chunk_builder
                     .append_row_matched(Op::Insert, &matched_row.row)
+                    .map(Self::post_process)
             } else {
                 None
             }
@@ -191,10 +236,12 @@ impl<const T: JoinTypePrimitive, const SIDE: SideTypePrimitive> JoinChunkBuilder
             }
             self.stream_chunk_builder
                 .append_row(Op::UpdateInsert, row, &matched_row.row)
+                .map(Self::post_process)
         // Inner sides
         } else {
             self.stream_chunk_builder
                 .append_row(Op::Insert, row, &matched_row.row)
+                .map(Self::post_process)
         }
     }
 
@@ -208,6 +255,7 @@ impl<const T: JoinTypePrimitive, const SIDE: SideTypePrimitive> JoinChunkBuilder
             if matched_row.is_zero_degree() && only_forward_matched_side(T, SIDE) {
                 self.stream_chunk_builder
                     .append_row_matched(Op::Insert, &matched_row.row)
+                    .map(Self::post_process)
             } else {
                 None
             }
@@ -216,6 +264,7 @@ impl<const T: JoinTypePrimitive, const SIDE: SideTypePrimitive> JoinChunkBuilder
             if matched_row.is_zero_degree() && only_forward_matched_side(T, SIDE) {
                 self.stream_chunk_builder
                     .append_row_matched(Op::Delete, &matched_row.row)
+                    .map(Self::post_process)
             } else {
                 None
             }
@@ -232,6 +281,8 @@ impl<const T: JoinTypePrimitive, const SIDE: SideTypePrimitive> JoinChunkBuilder
             }
             self.stream_chunk_builder
                 .append_row_matched(Op::UpdateInsert, &matched_row.row)
+                .map(|c: StreamChunk| Self::post_process(c))
+
         // Inner sides
         } else {
             // concat with the matched_row and append the new
@@ -241,6 +292,7 @@ impl<const T: JoinTypePrimitive, const SIDE: SideTypePrimitive> JoinChunkBuilder
             // the assumption for U+ after U-.
             self.stream_chunk_builder
                 .append_row(Op::Delete, row, &matched_row.row)
+                .map(Self::post_process)
         }
     }
 
@@ -252,7 +304,9 @@ impl<const T: JoinTypePrimitive, const SIDE: SideTypePrimitive> JoinChunkBuilder
     ) -> Option<StreamChunk> {
         // if it's a semi join and the side needs to be maintained.
         if is_semi(T) && forward_exactly_once(T, SIDE) {
-            self.stream_chunk_builder.append_row_update(op, row)
+            self.stream_chunk_builder
+                .append_row_update(op, row)
+                .map(Self::post_process)
         } else {
             None
         }
@@ -262,7 +316,9 @@ impl<const T: JoinTypePrimitive, const SIDE: SideTypePrimitive> JoinChunkBuilder
     pub fn forward_if_not_matched(&mut self, op: Op, row: RowRef<'_>) -> Option<StreamChunk> {
         // if it's outer join or anti join and the side needs to be maintained.
         if (is_anti(T) && forward_exactly_once(T, SIDE)) || is_outer_side(T, SIDE) {
-            self.stream_chunk_builder.append_row_update(op, row)
+            self.stream_chunk_builder
+                .append_row_update(op, row)
+                .map(Self::post_process)
         } else {
             None
         }
@@ -270,6 +326,6 @@ impl<const T: JoinTypePrimitive, const SIDE: SideTypePrimitive> JoinChunkBuilder
 
     #[inline]
     pub fn take(&mut self) -> Option<StreamChunk> {
-        self.stream_chunk_builder.take()
+        self.stream_chunk_builder.take().map(Self::post_process)
     }
 }
diff --git a/src/stream/src/executor/join/hash_join.rs b/src/stream/src/executor/join/hash_join.rs
index 6398a475a6062..10e9e26f784cd 100644
--- a/src/stream/src/executor/join/hash_join.rs
+++ b/src/stream/src/executor/join/hash_join.rs
@@ -22,7 +22,7 @@ use futures::future::{join, try_join};
 use futures::{pin_mut, stream, StreamExt};
 use futures_async_stream::for_await;
 use local_stats_alloc::{SharedStatsAlloc, StatsAlloc};
-use risingwave_common::buffer::Bitmap;
+use risingwave_common::bitmap::Bitmap;
 use risingwave_common::hash::{HashKey, PrecomputedBuildHasher};
 use risingwave_common::metrics::LabelGuardedIntCounter;
 use risingwave_common::row::{OwnedRow, Row, RowExt};
@@ -310,8 +310,8 @@ impl<K: HashKey, S: StateStore> JoinHashMap<K, S> {
 
     pub fn update_watermark(&mut self, watermark: ScalarImpl) {
         // TODO: remove data in cache.
-        self.state.table.update_watermark(watermark.clone(), false);
-        self.degree_state.table.update_watermark(watermark, false);
+        self.state.table.update_watermark(watermark.clone());
+        self.degree_state.table.update_watermark(watermark);
     }
 
     /// Take the state for the given `key` out of the hash table and return it. One **MUST** call
diff --git a/src/stream/src/executor/merge.rs b/src/stream/src/executor/merge.rs
index 19124fe8c22d4..8254dfeadc68b 100644
--- a/src/stream/src/executor/merge.rs
+++ b/src/stream/src/executor/merge.rs
@@ -74,20 +74,32 @@ impl MergeExecutor {
     }
 
     #[cfg(test)]
-    pub fn for_test(inputs: Vec<super::exchange::permit::Receiver>) -> Self {
+    pub fn for_test(
+        actor_id: ActorId,
+        inputs: Vec<super::exchange::permit::Receiver>,
+        shared_context: Arc<SharedContext>,
+    ) -> Self {
         use super::exchange::input::LocalInput;
         use crate::executor::exchange::input::Input;
 
         Self::new(
-            ActorContext::for_test(114),
+            ActorContext::for_test(actor_id),
             514,
             1919,
             inputs
                 .into_iter()
                 .enumerate()
-                .map(|(idx, input)| LocalInput::new(input, idx as ActorId).boxed_input())
+                .map(|(idx, input)| {
+                    LocalInput::new(
+                        input,
+                        idx as ActorId,
+                        actor_id,
+                        shared_context.local_barrier_manager.clone(),
+                    )
+                    .boxed_input()
+                })
                 .collect(),
-            SharedContext::for_test().into(),
+            shared_context,
             810,
             StreamingMetrics::unused().into(),
         )
@@ -474,10 +486,11 @@ mod tests {
     use tonic::{Request, Response, Status, Streaming};
 
     use super::*;
-    use crate::executor::exchange::input::RemoteInput;
+    use crate::executor::exchange::input::{Input, RemoteInput};
     use crate::executor::exchange::permit::channel_for_test;
+    use crate::executor::{BarrierInner as Barrier, MessageInner as Message};
+    use crate::task::barrier_test_utils::LocalBarrierTestEnv;
     use crate::task::test_utils::helper_make_local_actor;
-    use crate::task::LocalBarrierManager;
 
     fn build_test_chunk(epoch: u64) -> StreamChunk {
         // The number of items in `ops` is the epoch count.
@@ -495,64 +508,80 @@ mod tests {
             txs.push(tx);
             rxs.push(rx);
         }
-        let merger = MergeExecutor::for_test(rxs);
+        let barrier_test_env = LocalBarrierTestEnv::for_test().await;
+        let merger = MergeExecutor::for_test(233, rxs, barrier_test_env.shared_context.clone());
+        let actor_id = merger.actor_context.id;
         let mut handles = Vec::with_capacity(CHANNEL_NUMBER);
 
-        let epochs = (10..1000u64).step_by(10).collect_vec();
+        let epochs = (10..1000u64)
+            .step_by(10)
+            .map(|idx| (idx, test_epoch(idx)))
+            .collect_vec();
+        let mut prev_epoch = 0;
+        let prev_epoch = &mut prev_epoch;
+        let barriers: HashMap<_, _> = epochs
+            .iter()
+            .map(|(_, epoch)| {
+                let barrier = Barrier::with_prev_epoch_for_test(*epoch, *prev_epoch);
+                *prev_epoch = *epoch;
+                barrier_test_env.inject_barrier(&barrier, [], [actor_id]);
+                (*epoch, barrier)
+            })
+            .collect();
+        let b2 = Barrier::with_prev_epoch_for_test(test_epoch(1000), *prev_epoch)
+            .with_mutation(Mutation::Stop(HashSet::default()));
+        barrier_test_env.inject_barrier(&b2, [], [actor_id]);
 
         for (tx_id, tx) in txs.into_iter().enumerate() {
             let epochs = epochs.clone();
+            let barriers = barriers.clone();
+            let b2 = b2.clone();
             let handle = tokio::spawn(async move {
-                for epoch in epochs {
-                    if epoch % 20 == 0 {
-                        tx.send(Message::Chunk(build_test_chunk(epoch)))
+                for (idx, epoch) in epochs {
+                    if idx % 20 == 0 {
+                        tx.send(Message::Chunk(build_test_chunk(idx)))
                             .await
                             .unwrap();
                     } else {
                         tx.send(Message::Watermark(Watermark {
-                            col_idx: (epoch as usize / 20 + tx_id) % CHANNEL_NUMBER,
+                            col_idx: (idx as usize / 20 + tx_id) % CHANNEL_NUMBER,
                             data_type: DataType::Int64,
-                            val: ScalarImpl::Int64(epoch as i64),
+                            val: ScalarImpl::Int64(idx as i64),
                         }))
                         .await
                         .unwrap();
                     }
-                    tx.send(Message::Barrier(Barrier::new_test_barrier(test_epoch(
-                        epoch,
-                    ))))
-                    .await
-                    .unwrap();
+                    tx.send(Message::Barrier(barriers[&epoch].clone().into_dispatcher()))
+                        .await
+                        .unwrap();
                     sleep(Duration::from_millis(1)).await;
                 }
-                tx.send(Message::Barrier(
-                    Barrier::new_test_barrier(test_epoch(1000))
-                        .with_mutation(Mutation::Stop(HashSet::default())),
-                ))
-                .await
-                .unwrap();
+                tx.send(Message::Barrier(b2.clone().into_dispatcher()))
+                    .await
+                    .unwrap();
             });
             handles.push(handle);
         }
 
         let mut merger = merger.boxed().execute();
-        for epoch in epochs {
+        for (idx, epoch) in epochs {
             // expect n chunks
-            if epoch % 20 == 0 {
+            if idx % 20 == 0 {
                 for _ in 0..CHANNEL_NUMBER {
                     assert_matches!(merger.next().await.unwrap().unwrap(), Message::Chunk(chunk) => {
-                        assert_eq!(chunk.ops().len() as u64, epoch);
+                        assert_eq!(chunk.ops().len() as u64, idx);
                     });
                 }
-            } else if epoch as usize / 20 >= CHANNEL_NUMBER - 1 {
+            } else if idx as usize / 20 >= CHANNEL_NUMBER - 1 {
                 for _ in 0..CHANNEL_NUMBER {
                     assert_matches!(merger.next().await.unwrap().unwrap(), Message::Watermark(watermark) => {
-                        assert_eq!(watermark.val, ScalarImpl::Int64((epoch - 20 * (CHANNEL_NUMBER as u64 - 1)) as i64));
+                        assert_eq!(watermark.val, ScalarImpl::Int64((idx - 20 * (CHANNEL_NUMBER as u64 - 1)) as i64));
                     });
                 }
             }
             // expect a barrier
             assert_matches!(merger.next().await.unwrap().unwrap(), Message::Barrier(Barrier{epoch:barrier_epoch,mutation:_,..}) => {
-                assert_eq!(barrier_epoch.curr, test_epoch(epoch));
+                assert_eq!(barrier_epoch.curr, epoch);
             });
         }
         assert_matches!(
@@ -572,7 +601,8 @@ mod tests {
     async fn test_configuration_change() {
         let actor_id = 233;
         let (untouched, old, new) = (234, 235, 238); // upstream actors
-        let ctx = Arc::new(SharedContext::for_test());
+        let barrier_test_env = LocalBarrierTestEnv::for_test().await;
+        let ctx = barrier_test_env.shared_context.clone();
         let metrics = Arc::new(StreamingMetrics::unused());
 
         // 1. Register info in context.
@@ -628,9 +658,21 @@ mod tests {
                 }
             };
         }
+
+        macro_rules! assert_recv_pending {
+            () => {
+                assert!(merge
+                    .next()
+                    .now_or_never()
+                    .flatten()
+                    .transpose()
+                    .unwrap()
+                    .is_none());
+            };
+        }
         macro_rules! recv {
             () => {
-                merge.next().now_or_never().flatten().transpose().unwrap()
+                merge.next().await.transpose().unwrap()
             };
         }
 
@@ -638,7 +680,7 @@ mod tests {
         send!([untouched, old], Message::Chunk(StreamChunk::default()));
         recv!().unwrap().as_chunk().unwrap(); // We should be able to receive the chunk twice.
         recv!().unwrap().as_chunk().unwrap();
-        assert!(recv!().is_none());
+        assert_recv_pending!();
 
         // 4. Send a configuration change barrier.
         let merge_updates = maplit::hashmap! {
@@ -661,23 +703,31 @@ mod tests {
                 actor_new_dispatchers: Default::default(),
             },
         ));
-        send!([untouched, old], Message::Barrier(b1.clone()));
-        assert!(recv!().is_none()); // We should not receive the barrier, since merger is waiting for the new upstream new.
+        barrier_test_env.inject_barrier(&b1, [], [actor_id]);
+        send!(
+            [untouched, old],
+            Message::Barrier(b1.clone().into_dispatcher())
+        );
+        assert_recv_pending!(); // We should not receive the barrier, since merger is waiting for the new upstream new.
 
-        send!([new], Message::Barrier(b1.clone()));
+        send!([new], Message::Barrier(b1.clone().into_dispatcher()));
         recv!().unwrap().as_barrier().unwrap(); // We should now receive the barrier.
 
         // 5. Send a chunk.
         send!([untouched, new], Message::Chunk(StreamChunk::default()));
         recv!().unwrap().as_chunk().unwrap(); // We should be able to receive the chunk twice, since old is removed.
         recv!().unwrap().as_chunk().unwrap();
-        assert!(recv!().is_none());
+        assert_recv_pending!();
     }
 
     struct FakeExchangeService {
         rpc_called: Arc<AtomicBool>,
     }
 
+    fn exchange_client_test_barrier() -> crate::executor::Barrier {
+        Barrier::new_test_barrier(test_epoch(1))
+    }
+
     #[async_trait::async_trait]
     impl ExchangeService for FakeExchangeService {
         type GetDataStream = ReceiverStream<std::result::Result<GetDataResponse, Status>>;
@@ -711,7 +761,7 @@ mod tests {
             .await
             .unwrap();
             // send barrier
-            let barrier = Barrier::new_test_barrier(test_epoch(1));
+            let barrier = exchange_client_test_barrier();
             tx.send(Ok(GetStreamResponse {
                 message: Some(StreamMessage {
                     stream_message: Some(
@@ -755,10 +805,12 @@ mod tests {
         sleep(Duration::from_secs(1)).await;
         assert!(server_run.load(Ordering::SeqCst));
 
+        let test_env = LocalBarrierTestEnv::for_test().await;
+
         let remote_input = {
-            let pool = ComputeClientPool::default();
+            let pool = ComputeClientPool::for_test();
             RemoteInput::new(
-                LocalBarrierManager::for_test(),
+                test_env.shared_context.local_barrier_manager.clone(),
                 pool,
                 addr.into(),
                 (0, 0),
@@ -768,6 +820,12 @@ mod tests {
             )
         };
 
+        test_env.inject_barrier(
+            &exchange_client_test_barrier(),
+            [],
+            [remote_input.actor_id()],
+        );
+
         pin_mut!(remote_input);
 
         assert_matches!(remote_input.next().await.unwrap().unwrap(), Message::Chunk(chunk) => {
diff --git a/src/stream/src/executor/mod.rs b/src/stream/src/executor/mod.rs
index 36c6e8d173e3f..cfa8e895ad16f 100644
--- a/src/stream/src/executor/mod.rs
+++ b/src/stream/src/executor/mod.rs
@@ -24,7 +24,7 @@ use futures::stream::BoxStream;
 use futures::{Stream, StreamExt};
 use itertools::Itertools;
 use risingwave_common::array::StreamChunk;
-use risingwave_common::buffer::Bitmap;
+use risingwave_common::bitmap::Bitmap;
 use risingwave_common::catalog::{Schema, TableId};
 use risingwave_common::row::OwnedRow;
 use risingwave_common::types::{DataType, Datum, DefaultOrd, ScalarImpl};
@@ -40,10 +40,10 @@ use risingwave_pb::stream_plan::barrier_mutation::Mutation as PbMutation;
 use risingwave_pb::stream_plan::stream_message::StreamMessage;
 use risingwave_pb::stream_plan::update_mutation::{DispatcherUpdate, MergeUpdate};
 use risingwave_pb::stream_plan::{
-    BarrierMutation, CombinedMutation, CreateSubscriptionMutation, Dispatchers,
-    DropSubscriptionMutation, PauseMutation, PbAddMutation, PbBarrier, PbDispatcher,
-    PbStreamMessage, PbUpdateMutation, PbWatermark, ResumeMutation, SourceChangeSplitMutation,
-    StopMutation, ThrottleMutation,
+    BarrierMutation, CombinedMutation, Dispatchers, DropSubscriptionsMutation, PauseMutation,
+    PbAddMutation, PbBarrier, PbBarrierMutation, PbDispatcher, PbStreamMessage, PbUpdateMutation,
+    PbWatermark, ResumeMutation, SourceChangeSplitMutation, StopMutation, SubscriptionUpstreamInfo,
+    ThrottleMutation,
 };
 use smallvec::SmallVec;
 
@@ -77,6 +77,7 @@ mod lookup;
 mod lookup_union;
 mod merge;
 mod mview;
+mod nested_loop_temporal_join;
 mod no_op;
 mod now;
 mod over_window;
@@ -102,6 +103,10 @@ mod watermark;
 mod watermark_filter;
 mod wrapper;
 
+mod approx_percentile;
+
+mod row_merge;
+
 #[cfg(test)]
 mod integration_tests;
 pub mod test_utils;
@@ -109,6 +114,8 @@ mod utils;
 
 pub use actor::{Actor, ActorContext, ActorContextRef};
 use anyhow::Context;
+pub use approx_percentile::global::GlobalApproxPercentileExecutor;
+pub use approx_percentile::local::LocalApproxPercentileExecutor;
 pub use backfill::arrangement_backfill::*;
 pub use backfill::cdc::{CdcBackfillExecutor, CdcScanOptions, ExternalStorageTable};
 pub use backfill::no_shuffle_backfill::*;
@@ -138,11 +145,12 @@ pub use project_set::*;
 pub use rearranged_chain::RearrangedChainExecutor;
 pub use receiver::ReceiverExecutor;
 use risingwave_pb::source::{ConnectorSplit, ConnectorSplits};
+pub use row_merge::RowMergeExecutor;
 pub use simple_agg::SimpleAggExecutor;
 pub use sink::SinkExecutor;
 pub use sort::*;
 pub use stateless_simple_agg::StatelessSimpleAggExecutor;
-pub use temporal_join::*;
+pub use temporal_join::TemporalJoinExecutor;
 pub use top_n::{
     AppendOnlyGroupTopNExecutor, AppendOnlyTopNExecutor, GroupTopNExecutor, TopNExecutor,
 };
@@ -273,6 +281,8 @@ pub struct AddMutation {
     // TODO: remove this and use `SourceChangesSplit` after we support multiple mutations.
     pub splits: HashMap<ActorId, Vec<SplitImpl>>,
     pub pause: bool,
+    /// (`upstream_mv_table_id`,  `subscriber_id`)
+    pub subscriptions_to_add: Vec<(TableId, u32)>,
 }
 
 /// See [`PbMutation`] for the semantics of each mutation.
@@ -286,30 +296,34 @@ pub enum Mutation {
     Resume,
     Throttle(HashMap<ActorId, Option<u32>>),
     AddAndUpdate(AddMutation, UpdateMutation),
-    CreateSubscription {
-        subscription_id: u32,
-        upstream_mv_table_id: TableId,
-    },
-    DropSubscription {
-        subscription_id: u32,
-        upstream_mv_table_id: TableId,
+    DropSubscriptions {
+        /// `subscriber` -> `upstream_mv_table_id`
+        subscriptions_to_drop: Vec<(u32, TableId)>,
     },
 }
 
+/// The generic type `M` is the mutation type of the barrier.
+///
+/// For barrier of in the dispatcher, `M` is `()`, which means the mutation is erased.
+/// For barrier flowing within the streaming actor, `M` is the normal `BarrierMutationType`.
 #[derive(Debug, Clone)]
-pub struct Barrier {
+pub struct BarrierInner<M> {
     pub epoch: EpochPair,
-    pub mutation: Option<Arc<Mutation>>,
+    pub mutation: M,
     pub kind: BarrierKind,
 
     /// Tracing context for the **current** epoch of this barrier.
-    tracing_context: TracingContext,
+    pub tracing_context: TracingContext,
 
     /// The actors that this barrier has passed locally. Used for debugging only.
     pub passed_actors: Vec<ActorId>,
 }
 
-impl Barrier {
+pub type BarrierMutationType = Option<Arc<Mutation>>;
+pub type Barrier = BarrierInner<BarrierMutationType>;
+pub type DispatcherBarrier = BarrierInner<()>;
+
+impl<M: Default> BarrierInner<M> {
     /// Create a plain barrier.
     pub fn new_test_barrier(epoch: u64) -> Self {
         Self {
@@ -330,6 +344,18 @@ impl Barrier {
             passed_actors: Default::default(),
         }
     }
+}
+
+impl Barrier {
+    pub fn into_dispatcher(self) -> DispatcherBarrier {
+        DispatcherBarrier {
+            epoch: self.epoch,
+            mutation: (),
+            kind: self.kind,
+            tracing_context: self.tracing_context,
+            passed_actors: self.passed_actors,
+        }
+    }
 
     #[must_use]
     pub fn with_mutation(self, mutation: Mutation) -> Self {
@@ -425,8 +451,7 @@ impl Barrier {
             | Mutation::Resume
             | Mutation::SourceChangeSplit(_)
             | Mutation::Throttle(_)
-            | Mutation::CreateSubscription { .. }
-            | Mutation::DropSubscription { .. } => false,
+            | Mutation::DropSubscriptions { .. } => false,
         }
     }
 
@@ -436,6 +461,7 @@ impl Barrier {
             Some(
                   Mutation::Update { .. } // new actors for scaling
                 | Mutation::Add(AddMutation { pause: true, .. }) // new streaming job, or recovery
+                | Mutation::AddAndUpdate(AddMutation { pause: true, .. }, _) // new actors for replacing table
             ) => true,
             _ => false,
         }
@@ -490,9 +516,34 @@ impl Barrier {
     pub fn tracing_context(&self) -> &TracingContext {
         &self.tracing_context
     }
+
+    pub fn added_subscriber_on_mv_table(
+        &self,
+        mv_table_id: TableId,
+    ) -> impl Iterator<Item = u32> + '_ {
+        if let Some(Mutation::Add(add)) | Some(Mutation::AddAndUpdate(add, _)) =
+            self.mutation.as_deref()
+        {
+            Some(add)
+        } else {
+            None
+        }
+        .into_iter()
+        .flat_map(move |add| {
+            add.subscriptions_to_add.iter().filter_map(
+                move |(upstream_mv_table_id, subscriber_id)| {
+                    if *upstream_mv_table_id == mv_table_id {
+                        Some(*subscriber_id)
+                    } else {
+                        None
+                    }
+                },
+            )
+        })
+    }
 }
 
-impl PartialEq for Barrier {
+impl<M: PartialEq> PartialEq for BarrierInner<M> {
     fn eq(&self, other: &Self) -> bool {
         self.epoch == other.epoch && self.mutation == other.mutation
     }
@@ -559,6 +610,7 @@ impl Mutation {
                 added_actors,
                 splits,
                 pause,
+                subscriptions_to_add,
             }) => PbMutation::Add(PbAddMutation {
                 actor_dispatchers: adds
                     .iter()
@@ -574,6 +626,13 @@ impl Mutation {
                 added_actors: added_actors.iter().copied().collect(),
                 actor_splits: actor_splits_to_protobuf(splits),
                 pause: *pause,
+                subscriptions_to_add: subscriptions_to_add
+                    .iter()
+                    .map(|(table_id, subscriber_id)| SubscriptionUpstreamInfo {
+                        subscriber_id: *subscriber_id,
+                        upstream_mv_table_id: table_id.table_id,
+                    })
+                    .collect(),
             }),
             Mutation::SourceChangeSplit(changes) => PbMutation::Splits(SourceChangeSplitMutation {
                 actor_splits: changes
@@ -607,19 +666,18 @@ impl Mutation {
                     },
                 ],
             }),
-            Mutation::CreateSubscription {
-                upstream_mv_table_id,
-                subscription_id,
-            } => PbMutation::CreateSubscription(CreateSubscriptionMutation {
-                upstream_mv_table_id: upstream_mv_table_id.table_id,
-                subscription_id: *subscription_id,
-            }),
-            Mutation::DropSubscription {
-                upstream_mv_table_id,
-                subscription_id,
-            } => PbMutation::DropSubscription(DropSubscriptionMutation {
-                upstream_mv_table_id: upstream_mv_table_id.table_id,
-                subscription_id: *subscription_id,
+            Mutation::DropSubscriptions {
+                subscriptions_to_drop,
+            } => PbMutation::DropSubscriptions(DropSubscriptionsMutation {
+                info: subscriptions_to_drop
+                    .iter()
+                    .map(
+                        |(subscriber_id, upstream_mv_table_id)| SubscriptionUpstreamInfo {
+                            subscriber_id: *subscriber_id,
+                            upstream_mv_table_id: upstream_mv_table_id.table_id,
+                        },
+                    )
+                    .collect(),
             }),
         }
     }
@@ -690,6 +748,18 @@ impl Mutation {
                     })
                     .collect(),
                 pause: add.pause,
+                subscriptions_to_add: add
+                    .subscriptions_to_add
+                    .iter()
+                    .map(
+                        |SubscriptionUpstreamInfo {
+                             subscriber_id,
+                             upstream_mv_table_id,
+                         }| {
+                            (TableId::new(*upstream_mv_table_id), *subscriber_id)
+                        },
+                    )
+                    .collect(),
             }),
 
             PbMutation::Splits(s) => {
@@ -718,13 +788,12 @@ impl Mutation {
                     .map(|(actor_id, limit)| (*actor_id, limit.rate_limit))
                     .collect(),
             ),
-            PbMutation::CreateSubscription(create) => Mutation::CreateSubscription {
-                upstream_mv_table_id: TableId::new(create.upstream_mv_table_id),
-                subscription_id: create.subscription_id,
-            },
-            PbMutation::DropSubscription(drop) => Mutation::DropSubscription {
-                upstream_mv_table_id: TableId::new(drop.upstream_mv_table_id),
-                subscription_id: drop.subscription_id,
+            PbMutation::DropSubscriptions(drop) => Mutation::DropSubscriptions {
+                subscriptions_to_drop: drop
+                    .info
+                    .iter()
+                    .map(|info| (info.subscriber_id, TableId::new(info.upstream_mv_table_id)))
+                    .collect(),
             },
             PbMutation::Combined(CombinedMutation { mutations }) => match &mutations[..] {
                 [BarrierMutation {
@@ -750,50 +819,72 @@ impl Mutation {
     }
 }
 
-impl Barrier {
-    pub fn to_protobuf(&self) -> PbBarrier {
-        let Barrier {
+impl<M> BarrierInner<M> {
+    fn to_protobuf_inner(&self, barrier_fn: impl FnOnce(&M) -> Option<PbMutation>) -> PbBarrier {
+        let Self {
             epoch,
             mutation,
             kind,
             passed_actors,
             tracing_context,
             ..
-        } = self.clone();
+        } = self;
 
         PbBarrier {
             epoch: Some(PbEpoch {
                 curr: epoch.curr,
                 prev: epoch.prev,
             }),
-            mutation: mutation.map(|m| BarrierMutation {
-                mutation: Some(m.to_protobuf()),
+            mutation: Some(PbBarrierMutation {
+                mutation: barrier_fn(mutation),
             }),
             tracing_context: tracing_context.to_protobuf(),
-            kind: kind as _,
-            passed_actors,
+            kind: *kind as _,
+            passed_actors: passed_actors.clone(),
         }
     }
 
-    pub fn from_protobuf(prost: &PbBarrier) -> StreamExecutorResult<Self> {
-        let mutation = prost
-            .mutation
-            .as_ref()
-            .map(|m| Mutation::from_protobuf(m.mutation.as_ref().unwrap()))
-            .transpose()?
-            .map(Arc::new);
+    fn from_protobuf_inner(
+        prost: &PbBarrier,
+        mutation_from_pb: impl FnOnce(Option<&PbMutation>) -> StreamExecutorResult<M>,
+    ) -> StreamExecutorResult<Self> {
         let epoch = prost.get_epoch()?;
 
-        Ok(Barrier {
+        Ok(Self {
             kind: prost.kind(),
             epoch: EpochPair::new(epoch.curr, epoch.prev),
-            mutation,
+            mutation: mutation_from_pb(
+                prost
+                    .mutation
+                    .as_ref()
+                    .and_then(|mutation| mutation.mutation.as_ref()),
+            )?,
             passed_actors: prost.get_passed_actors().clone(),
             tracing_context: TracingContext::from_protobuf(&prost.tracing_context),
         })
     }
 }
 
+impl DispatcherBarrier {
+    pub fn to_protobuf(&self) -> PbBarrier {
+        self.to_protobuf_inner(|_| None)
+    }
+}
+
+impl Barrier {
+    pub fn to_protobuf(&self) -> PbBarrier {
+        self.to_protobuf_inner(|mutation| mutation.as_ref().map(|mutation| mutation.to_protobuf()))
+    }
+
+    pub fn from_protobuf(prost: &PbBarrier) -> StreamExecutorResult<Self> {
+        Self::from_protobuf_inner(prost, |mutation| {
+            mutation
+                .map(|m| Mutation::from_protobuf(m).map(Arc::new))
+                .transpose()
+        })
+    }
+}
+
 #[derive(Debug, PartialEq, Eq, Clone)]
 pub struct Watermark {
     pub col_idx: usize,
@@ -870,12 +961,15 @@ impl Watermark {
 }
 
 #[derive(Debug, EnumAsInner, PartialEq, Clone)]
-pub enum Message {
+pub enum MessageInner<M> {
     Chunk(StreamChunk),
-    Barrier(Barrier),
+    Barrier(BarrierInner<M>),
     Watermark(Watermark),
 }
 
+pub type Message = MessageInner<BarrierMutationType>;
+pub type DispatcherMessage = MessageInner<()>;
+
 impl From<StreamChunk> for Message {
     fn from(chunk: StreamChunk) -> Self {
         Message::Chunk(chunk)
@@ -909,7 +1003,9 @@ impl Message {
             }) if mutation.as_ref().unwrap().is_stop()
         )
     }
+}
 
+impl DispatcherMessage {
     pub fn to_protobuf(&self) -> PbStreamMessage {
         let prost = match self {
             Self::Chunk(stream_chunk) => {
@@ -926,10 +1022,21 @@ impl Message {
 
     pub fn from_protobuf(prost: &PbStreamMessage) -> StreamExecutorResult<Self> {
         let res = match prost.get_stream_message()? {
-            StreamMessage::StreamChunk(chunk) => Message::Chunk(StreamChunk::from_protobuf(chunk)?),
-            StreamMessage::Barrier(barrier) => Message::Barrier(Barrier::from_protobuf(barrier)?),
+            StreamMessage::StreamChunk(chunk) => Self::Chunk(StreamChunk::from_protobuf(chunk)?),
+            StreamMessage::Barrier(barrier) => Self::Barrier(
+                DispatcherBarrier::from_protobuf_inner(barrier, |mutation| {
+                    if mutation.is_some() {
+                        if cfg!(debug_assertions) {
+                            panic!("should not receive message of barrier with mutation");
+                        } else {
+                            warn!(?barrier, "receive message of barrier with mutation");
+                        }
+                    }
+                    Ok(())
+                })?,
+            ),
             StreamMessage::Watermark(watermark) => {
-                Message::Watermark(Watermark::from_protobuf(watermark)?)
+                Self::Watermark(Watermark::from_protobuf(watermark)?)
             }
         };
         Ok(res)
diff --git a/src/stream/src/executor/monitor/streaming_stats.rs b/src/stream/src/executor/monitor/streaming_stats.rs
index 379bdf49a5722..2cbd252d0bdb8 100644
--- a/src/stream/src/executor/monitor/streaming_stats.rs
+++ b/src/stream/src/executor/monitor/streaming_stats.rs
@@ -24,7 +24,7 @@ use risingwave_common::config::MetricLevel;
 use risingwave_common::metrics::{
     LabelGuardedGauge, LabelGuardedGaugeVec, LabelGuardedHistogramVec, LabelGuardedIntCounter,
     LabelGuardedIntCounterVec, LabelGuardedIntGauge, LabelGuardedIntGaugeVec,
-    RelabeledGuardedHistogramVec,
+    RelabeledGuardedHistogramVec, RelabeledGuardedIntCounterVec,
 };
 use risingwave_common::monitor::GLOBAL_METRICS_REGISTRY;
 use risingwave_common::{
@@ -96,7 +96,7 @@ pub struct StreamingMetrics {
     pub join_matched_join_keys: RelabeledGuardedHistogramVec<3>,
 
     // Streaming Join, Streaming Dynamic Filter and Streaming Union
-    pub barrier_align_duration: RelabeledGuardedHistogramVec<4>,
+    pub barrier_align_duration: RelabeledGuardedIntCounterVec<4>,
 
     // Streaming Aggregation
     agg_lookup_miss_count: LabelGuardedIntCounterVec<3>,
@@ -156,31 +156,32 @@ pub struct StreamingMetrics {
     pub barrier_manager_progress: IntCounter,
 
     // Sink related metrics
-    sink_commit_duration: LabelGuardedHistogramVec<3>,
-    connector_sink_rows_received: LabelGuardedIntCounterVec<2>,
-    log_store_first_write_epoch: LabelGuardedIntGaugeVec<3>,
-    log_store_latest_write_epoch: LabelGuardedIntGaugeVec<3>,
-    log_store_write_rows: LabelGuardedIntCounterVec<3>,
-    log_store_latest_read_epoch: LabelGuardedIntGaugeVec<3>,
-    log_store_read_rows: LabelGuardedIntCounterVec<3>,
-    log_store_reader_wait_new_future_duration_ns: LabelGuardedIntCounterVec<3>,
-    pub kv_log_store_storage_write_count: LabelGuardedIntCounterVec<3>,
-    pub kv_log_store_storage_write_size: LabelGuardedIntCounterVec<3>,
-    pub kv_log_store_rewind_count: LabelGuardedIntCounterVec<3>,
-    pub kv_log_store_rewind_delay: LabelGuardedHistogramVec<3>,
-    pub kv_log_store_storage_read_count: LabelGuardedIntCounterVec<4>,
-    pub kv_log_store_storage_read_size: LabelGuardedIntCounterVec<4>,
-    pub kv_log_store_buffer_unconsumed_item_count: LabelGuardedIntGaugeVec<3>,
-    pub kv_log_store_buffer_unconsumed_row_count: LabelGuardedIntGaugeVec<3>,
-    pub kv_log_store_buffer_unconsumed_epoch_count: LabelGuardedIntGaugeVec<3>,
-    pub kv_log_store_buffer_unconsumed_min_epoch: LabelGuardedIntGaugeVec<3>,
+    sink_commit_duration: LabelGuardedHistogramVec<4>,
+    connector_sink_rows_received: LabelGuardedIntCounterVec<3>,
+    log_store_first_write_epoch: LabelGuardedIntGaugeVec<4>,
+    log_store_latest_write_epoch: LabelGuardedIntGaugeVec<4>,
+    log_store_write_rows: LabelGuardedIntCounterVec<4>,
+    log_store_latest_read_epoch: LabelGuardedIntGaugeVec<4>,
+    log_store_read_rows: LabelGuardedIntCounterVec<4>,
+    log_store_reader_wait_new_future_duration_ns: LabelGuardedIntCounterVec<4>,
+
+    pub kv_log_store_storage_write_count: LabelGuardedIntCounterVec<4>,
+    pub kv_log_store_storage_write_size: LabelGuardedIntCounterVec<4>,
+    pub kv_log_store_rewind_count: LabelGuardedIntCounterVec<4>,
+    pub kv_log_store_rewind_delay: LabelGuardedHistogramVec<4>,
+    pub kv_log_store_storage_read_count: LabelGuardedIntCounterVec<5>,
+    pub kv_log_store_storage_read_size: LabelGuardedIntCounterVec<5>,
+    pub kv_log_store_buffer_unconsumed_item_count: LabelGuardedIntGaugeVec<4>,
+    pub kv_log_store_buffer_unconsumed_row_count: LabelGuardedIntGaugeVec<4>,
+    pub kv_log_store_buffer_unconsumed_epoch_count: LabelGuardedIntGaugeVec<4>,
+    pub kv_log_store_buffer_unconsumed_min_epoch: LabelGuardedIntGaugeVec<4>,
 
     // Sink iceberg metrics
-    iceberg_write_qps: LabelGuardedIntCounterVec<2>,
-    iceberg_write_latency: LabelGuardedHistogramVec<2>,
-    iceberg_rolling_unflushed_data_file: LabelGuardedIntGaugeVec<2>,
-    iceberg_position_delete_cache_num: LabelGuardedIntGaugeVec<2>,
-    iceberg_partition_num: LabelGuardedIntGaugeVec<2>,
+    iceberg_write_qps: LabelGuardedIntCounterVec<3>,
+    iceberg_write_latency: LabelGuardedHistogramVec<3>,
+    iceberg_rolling_unflushed_data_file: LabelGuardedIntGaugeVec<3>,
+    iceberg_position_delete_cache_num: LabelGuardedIntGaugeVec<3>,
+    iceberg_partition_num: LabelGuardedIntGaugeVec<3>,
 
     // Memory management
     pub lru_runtime_loop_count: IntCounter,
@@ -465,19 +466,15 @@ impl StreamingMetrics {
         )
         .unwrap();
 
-        let opts = histogram_opts!(
-            "stream_barrier_align_duration",
+        let barrier_align_duration = register_guarded_int_counter_vec_with_registry!(
+            "stream_barrier_align_duration_ns",
             "Duration of join align barrier",
-            exponential_buckets(0.0001, 2.0, 21).unwrap() // max 104s
-        );
-        let barrier_align_duration = register_guarded_histogram_vec_with_registry!(
-            opts,
             &["actor_id", "fragment_id", "wait_side", "executor"],
             registry
         )
         .unwrap();
 
-        let barrier_align_duration = RelabeledGuardedHistogramVec::with_metric_level_relabel_n(
+        let barrier_align_duration = RelabeledGuardedIntCounterVec::with_metric_level_relabel_n(
             MetricLevel::Debug,
             barrier_align_duration,
             level,
@@ -807,7 +804,7 @@ impl StreamingMetrics {
         let sink_commit_duration = register_guarded_histogram_vec_with_registry!(
             "sink_commit_duration",
             "Duration of commit op in sink",
-            &["executor_id", "connector", "sink_id"],
+            &["actor_id", "connector", "sink_id", "sink_name"],
             registry
         )
         .unwrap();
@@ -815,7 +812,7 @@ impl StreamingMetrics {
         let connector_sink_rows_received = register_guarded_int_counter_vec_with_registry!(
             "connector_sink_rows_received",
             "Number of rows received by sink",
-            &["connector_type", "sink_id"],
+            &["connector_type", "sink_id", "sink_name"],
             registry
         )
         .unwrap();
@@ -823,7 +820,7 @@ impl StreamingMetrics {
         let log_store_first_write_epoch = register_guarded_int_gauge_vec_with_registry!(
             "log_store_first_write_epoch",
             "The first write epoch of log store",
-            &["executor_id", "connector", "sink_id"],
+            &["actor_id", "connector", "sink_id", "sink_name"],
             registry
         )
         .unwrap();
@@ -831,7 +828,7 @@ impl StreamingMetrics {
         let log_store_latest_write_epoch = register_guarded_int_gauge_vec_with_registry!(
             "log_store_latest_write_epoch",
             "The latest write epoch of log store",
-            &["executor_id", "connector", "sink_id"],
+            &["actor_id", "connector", "sink_id", "sink_name"],
             registry
         )
         .unwrap();
@@ -839,7 +836,7 @@ impl StreamingMetrics {
         let log_store_write_rows = register_guarded_int_counter_vec_with_registry!(
             "log_store_write_rows",
             "The write rate of rows",
-            &["executor_id", "connector", "sink_id"],
+            &["actor_id", "connector", "sink_id", "sink_name"],
             registry
         )
         .unwrap();
@@ -847,7 +844,7 @@ impl StreamingMetrics {
         let log_store_latest_read_epoch = register_guarded_int_gauge_vec_with_registry!(
             "log_store_latest_read_epoch",
             "The latest read epoch of log store",
-            &["executor_id", "connector", "sink_id"],
+            &["actor_id", "connector", "sink_id", "sink_name"],
             registry
         )
         .unwrap();
@@ -855,7 +852,7 @@ impl StreamingMetrics {
         let log_store_read_rows = register_guarded_int_counter_vec_with_registry!(
             "log_store_read_rows",
             "The read rate of rows",
-            &["executor_id", "connector", "sink_id"],
+            &["actor_id", "connector", "sink_id", "sink_name"],
             registry
         )
         .unwrap();
@@ -864,7 +861,7 @@ impl StreamingMetrics {
             register_guarded_int_counter_vec_with_registry!(
                 "log_store_reader_wait_new_future_duration_ns",
                 "Accumulated duration of LogReader to wait for next call to create future",
-                &["executor_id", "connector", "sink_id"],
+                &["actor_id", "connector", "sink_id", "sink_name"],
                 registry
             )
             .unwrap();
@@ -872,7 +869,7 @@ impl StreamingMetrics {
         let kv_log_store_storage_write_count = register_guarded_int_counter_vec_with_registry!(
             "kv_log_store_storage_write_count",
             "Write row count throughput of kv log store",
-            &["executor_id", "connector", "sink_id"],
+            &["actor_id", "connector", "sink_id", "sink_name"],
             registry
         )
         .unwrap();
@@ -880,7 +877,7 @@ impl StreamingMetrics {
         let kv_log_store_storage_write_size = register_guarded_int_counter_vec_with_registry!(
             "kv_log_store_storage_write_size",
             "Write size throughput of kv log store",
-            &["executor_id", "connector", "sink_id"],
+            &["actor_id", "connector", "sink_id", "sink_name"],
             registry
         )
         .unwrap();
@@ -888,7 +885,7 @@ impl StreamingMetrics {
         let kv_log_store_storage_read_count = register_guarded_int_counter_vec_with_registry!(
             "kv_log_store_storage_read_count",
             "Write row count throughput of kv log store",
-            &["executor_id", "connector", "sink_id", "read_type"],
+            &["actor_id", "connector", "sink_id", "sink_name", "read_type"],
             registry
         )
         .unwrap();
@@ -896,7 +893,7 @@ impl StreamingMetrics {
         let kv_log_store_storage_read_size = register_guarded_int_counter_vec_with_registry!(
             "kv_log_store_storage_read_size",
             "Write size throughput of kv log store",
-            &["executor_id", "connector", "sink_id", "read_type"],
+            &["actor_id", "connector", "sink_id", "sink_name", "read_type"],
             registry
         )
         .unwrap();
@@ -904,7 +901,7 @@ impl StreamingMetrics {
         let kv_log_store_rewind_count = register_guarded_int_counter_vec_with_registry!(
             "kv_log_store_rewind_count",
             "Kv log store rewind rate",
-            &["executor_id", "connector", "sink_id"],
+            &["actor_id", "connector", "sink_id", "sink_name"],
             registry
         )
         .unwrap();
@@ -929,7 +926,7 @@ impl StreamingMetrics {
 
         let kv_log_store_rewind_delay = register_guarded_histogram_vec_with_registry!(
             kv_log_store_rewind_delay_opts,
-            &["executor_id", "connector", "sink_id"],
+            &["actor_id", "connector", "sink_id", "sink_name"],
             registry
         )
         .unwrap();
@@ -938,7 +935,7 @@ impl StreamingMetrics {
             register_guarded_int_gauge_vec_with_registry!(
                 "kv_log_store_buffer_unconsumed_item_count",
                 "Number of Unconsumed Item in buffer",
-                &["executor_id", "connector", "sink_id"],
+                &["actor_id", "connector", "sink_id", "sink_name"],
                 registry
             )
             .unwrap();
@@ -947,7 +944,7 @@ impl StreamingMetrics {
             register_guarded_int_gauge_vec_with_registry!(
                 "kv_log_store_buffer_unconsumed_row_count",
                 "Number of Unconsumed Row in buffer",
-                &["executor_id", "connector", "sink_id"],
+                &["actor_id", "connector", "sink_id", "sink_name"],
                 registry
             )
             .unwrap();
@@ -956,7 +953,7 @@ impl StreamingMetrics {
             register_guarded_int_gauge_vec_with_registry!(
                 "kv_log_store_buffer_unconsumed_epoch_count",
                 "Number of Unconsumed Epoch in buffer",
-                &["executor_id", "connector", "sink_id"],
+                &["actor_id", "connector", "sink_id", "sink_name"],
                 registry
             )
             .unwrap();
@@ -965,7 +962,7 @@ impl StreamingMetrics {
             register_guarded_int_gauge_vec_with_registry!(
                 "kv_log_store_buffer_unconsumed_min_epoch",
                 "Number of Unconsumed Epoch in buffer",
-                &["executor_id", "connector", "sink_id"],
+                &["actor_id", "connector", "sink_id", "sink_name"],
                 registry
             )
             .unwrap();
@@ -1067,7 +1064,7 @@ impl StreamingMetrics {
         let iceberg_write_qps = register_guarded_int_counter_vec_with_registry!(
             "iceberg_write_qps",
             "The qps of iceberg writer",
-            &["executor_id", "sink_id"],
+            &["actor_id", "sink_id", "sink_name"],
             registry
         )
         .unwrap();
@@ -1075,7 +1072,7 @@ impl StreamingMetrics {
         let iceberg_write_latency = register_guarded_histogram_vec_with_registry!(
             "iceberg_write_latency",
             "The latency of iceberg writer",
-            &["executor_id", "sink_id"],
+            &["actor_id", "sink_id", "sink_name"],
             registry
         )
         .unwrap();
@@ -1083,7 +1080,7 @@ impl StreamingMetrics {
         let iceberg_rolling_unflushed_data_file = register_guarded_int_gauge_vec_with_registry!(
             "iceberg_rolling_unflushed_data_file",
             "The unflushed data file count of iceberg rolling writer",
-            &["executor_id", "sink_id"],
+            &["actor_id", "sink_id", "sink_name"],
             registry
         )
         .unwrap();
@@ -1091,7 +1088,7 @@ impl StreamingMetrics {
         let iceberg_position_delete_cache_num = register_guarded_int_gauge_vec_with_registry!(
             "iceberg_position_delete_cache_num",
             "The delete cache num of iceberg position delete writer",
-            &["executor_id", "sink_id"],
+            &["actor_id", "sink_id", "sink_name"],
             registry
         )
         .unwrap();
@@ -1099,7 +1096,7 @@ impl StreamingMetrics {
         let iceberg_partition_num = register_guarded_int_gauge_vec_with_registry!(
             "iceberg_partition_num",
             "The partition num of iceberg partition writer",
-            &["executor_id", "sink_id"],
+            &["actor_id", "sink_id", "sink_name"],
             registry
         )
         .unwrap();
@@ -1221,17 +1218,19 @@ impl StreamingMetrics {
 
     pub fn new_sink_metrics(
         &self,
-        identity: &str,
+        actor_id_str: &str,
         sink_id_str: &str,
+        sink_name: &str,
         connector: &str,
     ) -> SinkMetrics {
-        let label_list = [identity, connector, sink_id_str];
+        let label_list = [actor_id_str, connector, sink_id_str, sink_name];
         let sink_commit_duration_metrics = self
             .sink_commit_duration
             .with_guarded_label_values(&label_list);
+
         let connector_sink_rows_received = self
             .connector_sink_rows_received
-            .with_guarded_label_values(&[connector, sink_id_str]);
+            .with_guarded_label_values(&[connector, sink_id_str, sink_name]);
 
         let log_store_latest_read_epoch = self
             .log_store_latest_read_epoch
@@ -1255,7 +1254,7 @@ impl StreamingMetrics {
             .log_store_reader_wait_new_future_duration_ns
             .with_guarded_label_values(&label_list);
 
-        let label_list = [identity, sink_id_str];
+        let label_list = [actor_id_str, sink_id_str, sink_name];
         let iceberg_write_qps = self
             .iceberg_write_qps
             .with_guarded_label_values(&label_list);
diff --git a/src/stream/src/executor/mview/materialize.rs b/src/stream/src/executor/mview/materialize.rs
index 1861313baddc6..9762d0ecd91ed 100644
--- a/src/stream/src/executor/mview/materialize.rs
+++ b/src/stream/src/executor/mview/materialize.rs
@@ -22,7 +22,7 @@ use bytes::Bytes;
 use futures::stream;
 use itertools::Itertools;
 use risingwave_common::array::Op;
-use risingwave_common::buffer::Bitmap;
+use risingwave_common::bitmap::Bitmap;
 use risingwave_common::catalog::{ColumnDesc, ColumnId, ConflictBehavior, TableId};
 use risingwave_common::row::{CompactedRow, RowDeserializer};
 use risingwave_common::types::DefaultOrd;
@@ -272,7 +272,7 @@ impl<S: StateStore, SD: ValueRowSerde> MaterializeExecutor<S, SD> {
                     }
                     Self::may_update_depended_subscriptions(
                         &mut self.depended_subscription_ids,
-                        b.mutation.as_deref(),
+                        &b,
                         mv_table_id,
                     );
                     let op_consistency_level = get_op_consistency_level(
@@ -305,51 +305,36 @@ impl<S: StateStore, SD: ValueRowSerde> MaterializeExecutor<S, SD> {
     /// return true when changed
     fn may_update_depended_subscriptions(
         depended_subscriptions: &mut HashSet<u32>,
-        mutation: Option<&Mutation>,
+        barrier: &Barrier,
         mv_table_id: TableId,
     ) {
-        let Some(mutation) = mutation else {
-            return;
-        };
-        match mutation {
-            Mutation::CreateSubscription {
-                subscription_id,
-                upstream_mv_table_id,
-            } => {
-                if *upstream_mv_table_id == mv_table_id
-                    && !depended_subscriptions.insert(*subscription_id)
-                {
-                    warn!(
-                        ?depended_subscriptions,
-                        ?mv_table_id,
-                        subscription_id,
-                        "subscription id already exists"
-                    );
-                }
+        for subscriber_id in barrier.added_subscriber_on_mv_table(mv_table_id) {
+            if !depended_subscriptions.insert(subscriber_id) {
+                warn!(
+                    ?depended_subscriptions,
+                    ?mv_table_id,
+                    subscriber_id,
+                    "subscription id already exists"
+                );
             }
-            Mutation::DropSubscription {
-                subscription_id,
-                upstream_mv_table_id,
-            } => {
+        }
+
+        if let Some(Mutation::DropSubscriptions {
+            subscriptions_to_drop,
+        }) = barrier.mutation.as_deref()
+        {
+            for (subscriber_id, upstream_mv_table_id) in subscriptions_to_drop {
                 if *upstream_mv_table_id == mv_table_id
-                    && !depended_subscriptions.remove(subscription_id)
+                    && !depended_subscriptions.remove(subscriber_id)
                 {
                     warn!(
                         ?depended_subscriptions,
                         ?mv_table_id,
-                        subscription_id,
-                        "drop non existing subscription id"
+                        subscriber_id,
+                        "drop non existing subscriber_id id"
                     );
                 }
             }
-            Mutation::Stop(_)
-            | Mutation::Update(_)
-            | Mutation::Add(_)
-            | Mutation::SourceChangeSplit(_)
-            | Mutation::Pause
-            | Mutation::Resume
-            | Mutation::Throttle(_)
-            | Mutation::AddAndUpdate(_, _) => {}
         }
     }
 }
diff --git a/src/stream/src/executor/nested_loop_temporal_join.rs b/src/stream/src/executor/nested_loop_temporal_join.rs
new file mode 100644
index 0000000000000..0888d8981fc8c
--- /dev/null
+++ b/src/stream/src/executor/nested_loop_temporal_join.rs
@@ -0,0 +1,276 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::HashMap;
+use std::sync::Arc;
+
+use futures::StreamExt;
+use futures_async_stream::try_stream;
+use risingwave_common::array::stream_chunk_builder::StreamChunkBuilder;
+use risingwave_common::array::StreamChunk;
+use risingwave_common::bitmap::BitmapBuilder;
+use risingwave_common::types::DataType;
+use risingwave_common::util::iter_util::ZipEqDebug;
+use risingwave_expr::expr::NonStrictExpression;
+use risingwave_hummock_sdk::{HummockEpoch, HummockReadEpoch};
+use risingwave_storage::store::PrefetchOptions;
+use risingwave_storage::table::batch_table::storage_table::StorageTable;
+use risingwave_storage::StateStore;
+
+use super::join::{JoinType, JoinTypePrimitive};
+use super::temporal_join::{align_input, apply_indices_map, phase1, InternalMessage};
+use super::{Execute, ExecutorInfo, Message, StreamExecutorError};
+use crate::common::metrics::MetricsInfo;
+use crate::executor::join::builder::JoinStreamChunkBuilder;
+use crate::executor::monitor::StreamingMetrics;
+use crate::executor::{ActorContextRef, Executor};
+
+pub struct NestedLoopTemporalJoinExecutor<S: StateStore, const T: JoinTypePrimitive> {
+    ctx: ActorContextRef,
+    #[allow(dead_code)]
+    info: ExecutorInfo,
+    left: Executor,
+    right: Executor,
+    right_table: TemporalSide<S>,
+    condition: Option<NonStrictExpression>,
+    output_indices: Vec<usize>,
+    chunk_size: usize,
+    // TODO: update metrics
+    #[allow(dead_code)]
+    metrics: Arc<StreamingMetrics>,
+}
+
+struct TemporalSide<S: StateStore> {
+    source: StorageTable<S>,
+}
+
+impl<S: StateStore> TemporalSide<S> {}
+
+#[try_stream(ok = StreamChunk, error = StreamExecutorError)]
+#[allow(clippy::too_many_arguments)]
+async fn phase1_handle_chunk<S: StateStore, E: phase1::Phase1Evaluation>(
+    chunk_size: usize,
+    right_size: usize,
+    full_schema: Vec<DataType>,
+    epoch: HummockEpoch,
+    right_table: &mut TemporalSide<S>,
+    chunk: StreamChunk,
+) {
+    let mut builder = StreamChunkBuilder::new(chunk_size, full_schema);
+
+    for (op, left_row) in chunk.rows() {
+        let mut matched = false;
+        #[for_await]
+        for keyed_row in right_table
+            .source
+            .batch_iter(
+                HummockReadEpoch::NoWait(epoch),
+                false,
+                PrefetchOptions::prefetch_for_large_range_scan(),
+            )
+            .await?
+        {
+            let keyed_row = keyed_row?;
+            let right_row = keyed_row.row();
+            matched = true;
+            if let Some(chunk) = E::append_matched_row(op, &mut builder, left_row, right_row) {
+                yield chunk;
+            }
+        }
+        if let Some(chunk) = E::match_end(&mut builder, op, left_row, right_size, matched) {
+            yield chunk;
+        }
+    }
+    if let Some(chunk) = builder.take() {
+        yield chunk;
+    }
+}
+
+impl<S: StateStore, const T: JoinTypePrimitive> NestedLoopTemporalJoinExecutor<S, T> {
+    #[allow(clippy::too_many_arguments)]
+    #[expect(dead_code)]
+    pub fn new(
+        ctx: ActorContextRef,
+        info: ExecutorInfo,
+        left: Executor,
+        right: Executor,
+        table: StorageTable<S>,
+        condition: Option<NonStrictExpression>,
+        output_indices: Vec<usize>,
+        metrics: Arc<StreamingMetrics>,
+        chunk_size: usize,
+    ) -> Self {
+        let _metrics_info = MetricsInfo::new(
+            metrics.clone(),
+            table.table_id().table_id,
+            ctx.id,
+            "nested loop temporal join",
+        );
+
+        Self {
+            ctx: ctx.clone(),
+            info,
+            left,
+            right,
+            right_table: TemporalSide { source: table },
+            condition,
+            output_indices,
+            chunk_size,
+            metrics,
+        }
+    }
+
+    #[try_stream(ok = Message, error = StreamExecutorError)]
+    async fn into_stream(mut self) {
+        let right_size = self.right.schema().len();
+
+        let (left_map, _right_map) = JoinStreamChunkBuilder::get_i2o_mapping(
+            &self.output_indices,
+            self.left.schema().len(),
+            right_size,
+        );
+
+        let left_to_output: HashMap<usize, usize> = HashMap::from_iter(left_map.iter().cloned());
+
+        let mut prev_epoch = None;
+
+        let full_schema: Vec<_> = self
+            .left
+            .schema()
+            .data_types()
+            .into_iter()
+            .chain(self.right.schema().data_types().into_iter())
+            .collect();
+
+        #[for_await]
+        for msg in align_input::<false>(self.left, self.right) {
+            match msg? {
+                InternalMessage::WaterMark(watermark) => {
+                    let output_watermark_col_idx = *left_to_output.get(&watermark.col_idx).unwrap();
+                    yield Message::Watermark(watermark.with_idx(output_watermark_col_idx));
+                }
+                InternalMessage::Chunk(chunk) => {
+                    let epoch = prev_epoch.expect("Chunk data should come after some barrier.");
+
+                    let full_schema = full_schema.clone();
+
+                    if T == JoinType::Inner {
+                        let st1 = phase1_handle_chunk::<S, phase1::Inner>(
+                            self.chunk_size,
+                            right_size,
+                            full_schema,
+                            epoch,
+                            &mut self.right_table,
+                            chunk,
+                        );
+                        #[for_await]
+                        for chunk in st1 {
+                            let chunk = chunk?;
+                            let new_chunk = if let Some(ref cond) = self.condition {
+                                let (data_chunk, ops) = chunk.into_parts();
+                                let passed_bitmap = cond.eval_infallible(&data_chunk).await;
+                                let passed_bitmap =
+                                    Arc::unwrap_or_clone(passed_bitmap).into_bool().to_bitmap();
+                                let (columns, vis) = data_chunk.into_parts();
+                                let new_vis = vis & passed_bitmap;
+                                StreamChunk::with_visibility(ops, columns, new_vis)
+                            } else {
+                                chunk
+                            };
+                            let new_chunk = apply_indices_map(new_chunk, &self.output_indices);
+                            yield Message::Chunk(new_chunk);
+                        }
+                    } else if let Some(ref cond) = self.condition {
+                        // Joined result without evaluating non-lookup conditions.
+                        let st1 = phase1_handle_chunk::<S, phase1::LeftOuterWithCond>(
+                            self.chunk_size,
+                            right_size,
+                            full_schema,
+                            epoch,
+                            &mut self.right_table,
+                            chunk,
+                        );
+                        let mut matched_count = 0usize;
+                        #[for_await]
+                        for chunk in st1 {
+                            let chunk = chunk?;
+                            let (data_chunk, ops) = chunk.into_parts();
+                            let passed_bitmap = cond.eval_infallible(&data_chunk).await;
+                            let passed_bitmap =
+                                Arc::unwrap_or_clone(passed_bitmap).into_bool().to_bitmap();
+                            let (columns, vis) = data_chunk.into_parts();
+                            let mut new_vis = BitmapBuilder::with_capacity(vis.len());
+                            for (passed, not_match_end) in
+                                passed_bitmap.iter().zip_eq_debug(vis.iter())
+                            {
+                                let is_match_end = !not_match_end;
+                                let vis = if is_match_end && matched_count == 0 {
+                                    // Nothing is matched, so the marker row should be visible.
+                                    true
+                                } else if is_match_end {
+                                    // reset the count
+                                    matched_count = 0;
+                                    // rows found, so the marker row should be invisible.
+                                    false
+                                } else {
+                                    if passed {
+                                        matched_count += 1;
+                                    }
+                                    passed
+                                };
+                                new_vis.append(vis);
+                            }
+                            let new_chunk = apply_indices_map(
+                                StreamChunk::with_visibility(ops, columns, new_vis.finish()),
+                                &self.output_indices,
+                            );
+                            yield Message::Chunk(new_chunk);
+                        }
+                        // The last row should always be marker row,
+                        assert_eq!(matched_count, 0);
+                    } else {
+                        let st1 = phase1_handle_chunk::<S, phase1::LeftOuter>(
+                            self.chunk_size,
+                            right_size,
+                            full_schema,
+                            epoch,
+                            &mut self.right_table,
+                            chunk,
+                        );
+                        #[for_await]
+                        for chunk in st1 {
+                            let chunk = chunk?;
+                            let new_chunk = apply_indices_map(chunk, &self.output_indices);
+                            yield Message::Chunk(new_chunk);
+                        }
+                    }
+                }
+                InternalMessage::Barrier(chunk, barrier) => {
+                    assert!(chunk.is_empty());
+                    if let Some(vnodes) = barrier.as_update_vnode_bitmap(self.ctx.id) {
+                        let _vnodes = self.right_table.source.update_vnode_bitmap(vnodes.clone());
+                    }
+                    prev_epoch = Some(barrier.epoch.curr);
+                    yield Message::Barrier(barrier)
+                }
+            }
+        }
+    }
+}
+
+impl<S: StateStore, const T: JoinTypePrimitive> Execute for NestedLoopTemporalJoinExecutor<S, T> {
+    fn execute(self: Box<Self>) -> super::BoxedMessageStream {
+        self.into_stream().boxed()
+    }
+}
diff --git a/src/stream/src/executor/rearranged_chain.rs b/src/stream/src/executor/rearranged_chain.rs
index 19ebfeabc2988..37717d270d90e 100644
--- a/src/stream/src/executor/rearranged_chain.rs
+++ b/src/stream/src/executor/rearranged_chain.rs
@@ -155,7 +155,7 @@ impl RearrangedChainExecutor {
                             // Update the progress since we've consumed all chunks before this
                             // phantom.
                             self.progress.update(
-                                last_rearranged_epoch.curr,
+                                last_rearranged_epoch,
                                 barrier.epoch.curr,
                                 processed_rows,
                             );
@@ -201,7 +201,7 @@ impl RearrangedChainExecutor {
                         continue;
                     };
                     if let Some(barrier) = msg.as_barrier() {
-                        self.progress.finish(barrier.epoch.curr, processed_rows);
+                        self.progress.finish(barrier.epoch, processed_rows);
                     }
                     yield msg;
                 }
@@ -214,7 +214,7 @@ impl RearrangedChainExecutor {
             for msg in upstream {
                 let msg: Message = msg?;
                 if let Some(barrier) = msg.as_barrier() {
-                    self.progress.finish(barrier.epoch.curr, processed_rows);
+                    self.progress.finish(barrier.epoch, processed_rows);
                 }
                 yield msg;
             }
diff --git a/src/stream/src/executor/receiver.rs b/src/stream/src/executor/receiver.rs
index effe773d54594..58700d2a13509 100644
--- a/src/stream/src/executor/receiver.rs
+++ b/src/stream/src/executor/receiver.rs
@@ -72,16 +72,26 @@ impl ReceiverExecutor {
     }
 
     #[cfg(test)]
-    pub fn for_test(input: super::exchange::permit::Receiver) -> Self {
+    pub fn for_test(
+        actor_id: ActorId,
+        input: super::exchange::permit::Receiver,
+        shared_context: Arc<SharedContext>,
+    ) -> Self {
         use super::exchange::input::LocalInput;
         use crate::executor::exchange::input::Input;
 
         Self::new(
-            ActorContext::for_test(114),
+            ActorContext::for_test(actor_id),
             514,
             1919,
-            LocalInput::new(input, 0).boxed_input(),
-            SharedContext::for_test().into(),
+            LocalInput::new(
+                input,
+                0,
+                actor_id,
+                shared_context.local_barrier_manager.clone(),
+            )
+            .boxed_input(),
+            shared_context,
             810,
             StreamingMetrics::unused().into(),
         )
@@ -194,7 +204,8 @@ mod tests {
     use risingwave_pb::stream_plan::update_mutation::MergeUpdate;
 
     use super::*;
-    use crate::executor::UpdateMutation;
+    use crate::executor::{MessageInner as Message, UpdateMutation};
+    use crate::task::barrier_test_utils::LocalBarrierTestEnv;
     use crate::task::test_utils::helper_make_local_actor;
 
     #[tokio::test]
@@ -202,7 +213,9 @@ mod tests {
         let actor_id = 233;
         let (old, new) = (114, 514); // old and new upstream actor id
 
-        let ctx = Arc::new(SharedContext::for_test());
+        let barrier_test_env = LocalBarrierTestEnv::for_test().await;
+
+        let ctx = barrier_test_env.shared_context.clone();
         let metrics = Arc::new(StreamingMetrics::unused());
 
         // 1. Register info in context.
@@ -261,21 +274,28 @@ mod tests {
                 }
             };
         }
-        macro_rules! recv {
+        macro_rules! assert_recv_pending {
             () => {
-                receiver
+                assert!(receiver
                     .next()
                     .now_or_never()
                     .flatten()
                     .transpose()
                     .unwrap()
+                    .is_none());
+            };
+        }
+
+        macro_rules! recv {
+            () => {
+                receiver.next().await.transpose().unwrap()
             };
         }
 
         // 3. Send a chunk.
         send!([old], Message::Chunk(StreamChunk::default()));
         recv!().unwrap().as_chunk().unwrap(); // We should be able to receive the chunk.
-        assert!(recv!().is_none());
+        assert_recv_pending!();
 
         // 4. Send a configuration change barrier.
         let merge_updates = maplit::hashmap! {
@@ -298,19 +318,22 @@ mod tests {
                 actor_new_dispatchers: Default::default(),
             },
         ));
-        send!([new], Message::Barrier(b1.clone()));
-        assert!(recv!().is_none()); // We should not receive the barrier, as new is not the upstream.
 
-        send!([old], Message::Barrier(b1.clone()));
+        barrier_test_env.inject_barrier(&b1, [], [actor_id]);
+
+        send!([new], Message::Barrier(b1.clone().into_dispatcher()));
+        assert_recv_pending!(); // We should not receive the barrier, as new is not the upstream.
+
+        send!([old], Message::Barrier(b1.clone().into_dispatcher()));
         recv!().unwrap().as_barrier().unwrap(); // We should now receive the barrier.
 
         // 5. Send a chunk to the removed upstream.
         send_error!([old], Message::Chunk(StreamChunk::default()));
-        assert!(recv!().is_none());
+        assert_recv_pending!();
 
         // 6. Send a chunk to the added upstream.
         send!([new], Message::Chunk(StreamChunk::default()));
         recv!().unwrap().as_chunk().unwrap(); // We should be able to receive the chunk.
-        assert!(recv!().is_none());
+        assert_recv_pending!();
     }
 }
diff --git a/src/stream/src/executor/row_id_gen.rs b/src/stream/src/executor/row_id_gen.rs
index 2da75389fd9f7..1fcb85c26f88e 100644
--- a/src/stream/src/executor/row_id_gen.rs
+++ b/src/stream/src/executor/row_id_gen.rs
@@ -14,7 +14,7 @@
 
 use risingwave_common::array::stream_chunk::Ops;
 use risingwave_common::array::{Array, ArrayBuilder, ArrayRef, Op, SerialArrayBuilder};
-use risingwave_common::buffer::Bitmap;
+use risingwave_common::bitmap::Bitmap;
 use risingwave_common::hash::VnodeBitmapExt;
 use risingwave_common::types::Serial;
 use risingwave_common::util::iter_util::ZipEqFast;
diff --git a/src/stream/src/executor/row_merge.rs b/src/stream/src/executor/row_merge.rs
new file mode 100644
index 0000000000000..b19b9bdcca8c8
--- /dev/null
+++ b/src/stream/src/executor/row_merge.rs
@@ -0,0 +1,189 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use risingwave_common::bail;
+use risingwave_common::types::ToOwnedDatum;
+use risingwave_common::util::chunk_coalesce::DataChunkBuilder;
+use risingwave_common::util::column_index_mapping::ColIndexMapping;
+
+use super::barrier_align::*;
+use crate::executor::prelude::*;
+
+pub struct RowMergeExecutor {
+    ctx: ActorContextRef,
+    pub lhs_input: Executor,
+    pub rhs_input: Executor,
+    /// Maps input from the lhs to the output.
+    pub lhs_mapping: ColIndexMapping,
+    /// Maps input from the rhs to the output.
+    pub rhs_mapping: ColIndexMapping,
+    /// Output schema
+    pub schema: Schema,
+}
+
+impl RowMergeExecutor {
+    pub fn new(
+        ctx: ActorContextRef,
+        lhs_input: Executor,
+        rhs_input: Executor,
+        lhs_mapping: ColIndexMapping,
+        rhs_mapping: ColIndexMapping,
+        schema: Schema,
+    ) -> Self {
+        Self {
+            ctx,
+            lhs_input,
+            rhs_input,
+            lhs_mapping,
+            rhs_mapping,
+            schema,
+        }
+    }
+
+    #[try_stream(ok = Message, error = StreamExecutorError)]
+    pub async fn execute_inner(self) {
+        let lhs_mapping = self.lhs_mapping;
+        let rhs_mapping = self.rhs_mapping;
+        let data_types = self
+            .schema
+            .fields()
+            .iter()
+            .map(|f| f.data_type())
+            .collect::<Vec<_>>();
+
+        {
+            let mut lhs_buffer: Vec<StreamChunk> = Vec::with_capacity(1);
+            let mut rhs_buffer: Vec<StreamChunk> = Vec::with_capacity(1);
+            let aligned_stream = barrier_align(
+                self.lhs_input.execute(),
+                self.rhs_input.execute(),
+                self.ctx.id,
+                self.ctx.fragment_id,
+                self.ctx.streaming_metrics.clone(),
+                "RowMerge",
+            );
+            pin_mut!(aligned_stream);
+            #[for_await]
+            for message in aligned_stream {
+                match message? {
+                    AlignedMessage::Left(chunk) => {
+                        lhs_buffer.push(chunk);
+                    }
+                    AlignedMessage::Right(chunk) => {
+                        rhs_buffer.push(chunk);
+                    }
+                    AlignedMessage::Barrier(barrier) => {
+                        if lhs_buffer.is_empty() && rhs_buffer.is_empty() {
+                            yield Message::Barrier(barrier);
+                            continue;
+                        }
+                        #[for_await]
+                        for output in Self::flush_buffers(
+                            &data_types,
+                            &lhs_mapping,
+                            &rhs_mapping,
+                            &mut lhs_buffer,
+                            &mut rhs_buffer,
+                        ) {
+                            yield output?;
+                        }
+                        yield Message::Barrier(barrier);
+                    }
+                    AlignedMessage::WatermarkLeft(watermark) => {
+                        tracing::warn!("unexpected watermark from left stream: {:?}", watermark);
+                    }
+                    AlignedMessage::WatermarkRight(watermark) => {
+                        tracing::warn!("unexpected watermark from right stream: {:?}", watermark);
+                    }
+                }
+            }
+        }
+    }
+
+    #[try_stream(ok = Message, error = StreamExecutorError)]
+    async fn flush_buffers<'a>(
+        data_types: &'a [DataType],
+        lhs_mapping: &'a ColIndexMapping,
+        rhs_mapping: &'a ColIndexMapping,
+        lhs_buffer: &'a mut Vec<StreamChunk>,
+        rhs_buffer: &'a mut Vec<StreamChunk>,
+    ) {
+        if lhs_buffer.is_empty() {
+            bail!("lhs buffer should not be empty ");
+        };
+        if rhs_buffer.is_empty() {
+            bail!("rhs buffer should not be empty ");
+        };
+
+        for lhs_chunk in lhs_buffer.drain(..) {
+            for rhs_chunk in rhs_buffer.drain(..) {
+                yield Self::build_chunk(
+                    data_types,
+                    lhs_mapping,
+                    rhs_mapping,
+                    lhs_chunk.clone(),
+                    rhs_chunk,
+                )?;
+            }
+        }
+    }
+
+    fn build_chunk(
+        data_types: &[DataType],
+        lhs_mapping: &ColIndexMapping,
+        rhs_mapping: &ColIndexMapping,
+        lhs_chunk: StreamChunk,
+        rhs_chunk: StreamChunk,
+    ) -> Result<Message, StreamExecutorError> {
+        if !(1..=2).contains(&lhs_chunk.cardinality()) {
+            bail!("lhs chunk cardinality should be 1 or 2");
+        }
+        if !(1..=2).contains(&rhs_chunk.cardinality()) {
+            bail!("rhs chunk cardinality should be 1 or 2");
+        }
+        if lhs_chunk.cardinality() != rhs_chunk.cardinality() {
+            bail!("lhs and rhs chunk cardinality should be the same");
+        }
+        let cardinality = lhs_chunk.cardinality();
+        let mut ops = Vec::with_capacity(cardinality);
+        let mut merged_rows = vec![vec![Datum::None; data_types.len()]; cardinality];
+        for (i, (op, lhs_row)) in lhs_chunk.rows().enumerate() {
+            ops.push(op);
+            for (j, d) in lhs_row.iter().enumerate() {
+                let out_index = lhs_mapping.map(j);
+                merged_rows[i][out_index] = d.to_owned_datum();
+            }
+        }
+
+        for (i, (_, rhs_row)) in rhs_chunk.rows().enumerate() {
+            for (j, d) in rhs_row.iter().enumerate() {
+                let out_index = rhs_mapping.map(j);
+                merged_rows[i][out_index] = d.to_owned_datum();
+            }
+        }
+        let mut builder = DataChunkBuilder::new(data_types.to_vec(), cardinality);
+        for row in merged_rows {
+            if let Some(chunk) = builder.append_one_row(&row[..]) {
+                return Ok(Message::Chunk(StreamChunk::from_parts(ops, chunk)));
+            }
+        }
+        bail!("builder should have yielded a chunk")
+    }
+}
+
+impl Execute for RowMergeExecutor {
+    fn execute(self: Box<Self>) -> BoxedMessageStream {
+        self.execute_inner().boxed()
+    }
+}
diff --git a/src/stream/src/executor/simple_agg.rs b/src/stream/src/executor/simple_agg.rs
index 3ed0c2533e45d..a08049268e5b4 100644
--- a/src/stream/src/executor/simple_agg.rs
+++ b/src/stream/src/executor/simple_agg.rs
@@ -186,7 +186,7 @@ impl<S: StateStore> SimpleAggExecutor<S> {
         vars: &mut ExecutionVars<S>,
         epoch: EpochPair,
     ) -> StreamExecutorResult<Option<StreamChunk>> {
-        let chunk = if vars.state_changed || vars.agg_group.is_uninitialized() {
+        if vars.state_changed || vars.agg_group.is_uninitialized() {
             // Flush distinct dedup state.
             vars.distinct_dedup.flush(&mut this.distinct_dedup_tables)?;
 
@@ -194,16 +194,14 @@ impl<S: StateStore> SimpleAggExecutor<S> {
             let encoded_states = vars.agg_group.encode_states(&this.agg_funcs)?;
             this.intermediate_state_table
                 .update_without_old_value(encoded_states);
+        }
 
-            // Retrieve modified states and put the changes into the builders.
-            vars.agg_group
-                .build_change(&this.storages, &this.agg_funcs)
-                .await?
-                .map(|change| change.to_stream_chunk(&this.info.schema.data_types()))
-        } else {
-            // No state is changed.
-            None
-        };
+        // Retrieve modified states and put the changes into the builders.
+        let chunk = vars
+            .agg_group
+            .build_change(&this.storages, &this.agg_funcs)
+            .await?
+            .map(|change| change.to_stream_chunk(&this.info.schema.data_types()));
 
         // Commit all state tables.
         futures::future::try_join_all(this.all_state_tables_mut().map(|table| table.commit(epoch)))
@@ -390,4 +388,97 @@ mod tests {
             )
         );
     }
+
+    // NOTE(kwannoel): `approx_percentile` + `keyed_merge` depend on this property for correctness.
+    #[tokio::test]
+    async fn test_simple_aggregation_always_output_per_epoch() {
+        let store = MemoryStateStore::new();
+        let schema = Schema {
+            fields: vec![
+                Field::unnamed(DataType::Int64),
+                Field::unnamed(DataType::Int64),
+                // primary key column`
+                Field::unnamed(DataType::Int64),
+            ],
+        };
+        let (mut tx, source) = MockSource::channel();
+        let source = source.into_executor(schema, vec![2]);
+        // initial barrier
+        tx.push_barrier(test_epoch(1), false);
+        // next barrier
+        tx.push_barrier(test_epoch(2), false);
+        tx.push_chunk(StreamChunk::from_pretty(
+            "   I   I    I
+            + 100 200 1001
+            - 100 200 1001",
+        ));
+        tx.push_barrier(test_epoch(3), false);
+        tx.push_barrier(test_epoch(4), false);
+
+        let agg_calls = vec![
+            AggCall::from_pretty("(count:int8)"),
+            AggCall::from_pretty("(sum:int8 $0:int8)"),
+            AggCall::from_pretty("(sum:int8 $1:int8)"),
+            AggCall::from_pretty("(min:int8 $0:int8)"),
+        ];
+
+        let simple_agg = new_boxed_simple_agg_executor(
+            ActorContext::for_test(123),
+            store,
+            source,
+            false,
+            agg_calls,
+            0,
+            vec![2],
+            1,
+        )
+        .await;
+        let mut simple_agg = simple_agg.execute();
+
+        // Consume the init barrier
+        simple_agg.next().await.unwrap().unwrap();
+        // Consume stream chunk
+        let msg = simple_agg.next().await.unwrap().unwrap();
+        assert_eq!(
+            *msg.as_chunk().unwrap(),
+            StreamChunk::from_pretty(
+                " I   I   I  I
+                + 0   .   .  . "
+            )
+        );
+        assert_matches!(
+            simple_agg.next().await.unwrap().unwrap(),
+            Message::Barrier { .. }
+        );
+
+        // Consume stream chunk
+        let msg = simple_agg.next().await.unwrap().unwrap();
+        assert_eq!(
+            *msg.as_chunk().unwrap(),
+            StreamChunk::from_pretty(
+                "  I   I   I  I
+                U- 0   .   .  .
+                U+ 0   .   .  ."
+            )
+        );
+        assert_matches!(
+            simple_agg.next().await.unwrap().unwrap(),
+            Message::Barrier { .. }
+        );
+
+        // Consume stream chunk
+        let msg = simple_agg.next().await.unwrap().unwrap();
+        assert_eq!(
+            *msg.as_chunk().unwrap(),
+            StreamChunk::from_pretty(
+                "  I   I   I  I
+                U- 0   .   .  .
+                U+ 0   .   .  ."
+            )
+        );
+        assert_matches!(
+            simple_agg.next().await.unwrap().unwrap(),
+            Message::Barrier { .. }
+        );
+    }
 }
diff --git a/src/stream/src/executor/sink.rs b/src/stream/src/executor/sink.rs
index 9b6c84f674cfd..ef8b57781cd95 100644
--- a/src/stream/src/executor/sink.rs
+++ b/src/stream/src/executor/sink.rs
@@ -93,7 +93,8 @@ impl<F: LogStoreFactory> SinkExecutor<F> {
         chunk_size: usize,
         input_data_types: Vec<DataType>,
     ) -> StreamExecutorResult<Self> {
-        let sink = build_sink(sink_param.clone())?;
+        let sink = build_sink(sink_param.clone())
+            .map_err(|e| StreamExecutorError::from((e, sink_param.sink_id.sink_id)))?;
         let sink_input_schema: Schema = columns
             .iter()
             .map(|column| Field::from(&column.column_desc))
@@ -362,20 +363,32 @@ impl<F: LogStoreFactory> SinkExecutor<F> {
                         } else {
                             chunks
                         };
-                        let chunks = if re_construct_with_sink_pk {
-                            StreamChunkCompactor::new(down_stream_pk.clone(), chunks)
+                        if re_construct_with_sink_pk {
+                            let chunks = StreamChunkCompactor::new(down_stream_pk.clone(), chunks)
                                 .reconstructed_compacted_chunks(
                                     chunk_size,
                                     input_data_types.clone(),
                                     sink_type != SinkType::ForceAppendOnly,
-                                )
+                                );
+                            for c in chunks {
+                                yield Message::Chunk(c);
+                            }
                         } else {
-                            chunks
+                            let mut chunk_builder =
+                                StreamChunkBuilder::new(chunk_size, input_data_types.clone());
+                            for chunk in chunks {
+                                for (op, row) in chunk.rows() {
+                                    if let Some(c) = chunk_builder.append_row(op, row) {
+                                        yield Message::Chunk(c);
+                                    }
+                                }
+                            }
+
+                            if let Some(c) = chunk_builder.take() {
+                                yield Message::Chunk(c);
+                            }
                         };
 
-                        for c in chunks {
-                            yield Message::Chunk(c);
-                        }
                         if let Some(w) = mem::take(&mut watermark) {
                             yield Message::Watermark(w)
                         }
@@ -469,6 +482,7 @@ impl<F: LogStoreFactory> SinkExecutor<F> {
                     warn!(
                         error = %e.as_report(),
                         executor_id = sink_writer_param.executor_id,
+                        sink_id = sink_param.sink_id.sink_id,
                         "rewind successfully after sink error"
                     );
                     sink_writer_param.vnode_bitmap = curr_vnode_bitmap;
@@ -482,7 +496,8 @@ impl<F: LogStoreFactory> SinkExecutor<F> {
                     );
                     Err(e)
                 }
-            }?;
+            }
+            .map_err(|e| StreamExecutorError::from((e, sink_param.sink_id.sink_id)))?;
         }
         Err(anyhow!("end of stream").into())
     }
@@ -563,6 +578,7 @@ mod test {
             sink_id: 0.into(),
             sink_name: "test".into(),
             properties,
+
             columns: columns
                 .iter()
                 .filter(|col| !col.is_hidden)
@@ -691,6 +707,7 @@ mod test {
             sink_id: 0.into(),
             sink_name: "test".into(),
             properties,
+
             columns: columns
                 .iter()
                 .filter(|col| !col.is_hidden)
@@ -792,6 +809,7 @@ mod test {
             sink_id: 0.into(),
             sink_name: "test".into(),
             properties,
+
             columns: columns
                 .iter()
                 .filter(|col| !col.is_hidden)
diff --git a/src/stream/src/executor/sort_buffer.rs b/src/stream/src/executor/sort_buffer.rs
index fc4f921e7ccbd..b3d658f3af4a5 100644
--- a/src/stream/src/executor/sort_buffer.rs
+++ b/src/stream/src/executor/sort_buffer.rs
@@ -34,7 +34,7 @@ use risingwave_storage::table::KeyedRow;
 use risingwave_storage::StateStore;
 
 use super::{StreamExecutorError, StreamExecutorResult};
-use crate::common::cache::{StateCache, StateCacheFiller, TopNStateCache};
+use crate::common::state_cache::{StateCache, StateCacheFiller, TopNStateCache};
 use crate::common::table::state_table::StateTable;
 
 type CacheKey = (
@@ -177,8 +177,8 @@ impl<S: StateStore> SortBuffer<S> {
         }
 
         // TODO(rc): Need something like `table.range_delete()`. Here we call
-        // `update_watermark(watermark, true)` as an alternative to `range_delete((..watermark))`.
-        buffer_table.update_watermark(watermark, true);
+        // `update_watermark(watermark)` as an alternative to `range_delete((..watermark))`.
+        buffer_table.update_watermark(watermark);
     }
 
     #[try_stream(ok = OwnedRow, error = StreamExecutorError)]
diff --git a/src/stream/src/executor/source/fetch_executor.rs b/src/stream/src/executor/source/fetch_executor.rs
index b4c006469e650..788a9a45662cd 100644
--- a/src/stream/src/executor/source/fetch_executor.rs
+++ b/src/stream/src/executor/source/fetch_executor.rs
@@ -199,7 +199,6 @@ impl<S: StateStore, Src: OpendalSource> FsFetchExecutor<S, Src> {
         else {
             unreachable!("Partition and offset columns must be set.");
         };
-
         // Initialize state table.
         state_store_handler.init_epoch(barrier.epoch);
 
diff --git a/src/stream/src/executor/source/list_executor.rs b/src/stream/src/executor/source/list_executor.rs
index 9317cfbc9aa4a..25b32c0a0e4b8 100644
--- a/src/stream/src/executor/source/list_executor.rs
+++ b/src/stream/src/executor/source/list_executor.rs
@@ -146,36 +146,45 @@ impl<S: StateStore> FsListExecutor<S> {
 
         yield Message::Barrier(barrier);
 
-        while let Some(msg) = stream.next().await {
-            match msg {
-                Err(e) => {
-                    tracing::warn!(error = %e.as_report(), "encountered an error, recovering");
-                    // todo: rebuild stream here
-                }
-                Ok(msg) => match msg {
-                    // Barrier arrives.
-                    Either::Left(msg) => match &msg {
-                        Message::Barrier(barrier) => {
-                            if let Some(mutation) = barrier.mutation.as_deref() {
-                                match mutation {
-                                    Mutation::Pause => stream.pause_stream(),
-                                    Mutation::Resume => stream.resume_stream(),
-                                    _ => (),
+        loop {
+            // a list file stream never ends, keep list to find if there is any new file.
+            while let Some(msg) = stream.next().await {
+                match msg {
+                    Err(e) => {
+                        tracing::warn!(error = %e.as_report(), "encountered an error, recovering");
+                        stream
+                            .replace_data_stream(self.build_chunked_paginate_stream(&source_desc)?);
+                    }
+                    Ok(msg) => match msg {
+                        // Barrier arrives.
+                        Either::Left(msg) => match &msg {
+                            Message::Barrier(barrier) => {
+                                if let Some(mutation) = barrier.mutation.as_deref() {
+                                    match mutation {
+                                        Mutation::Pause => stream.pause_stream(),
+                                        Mutation::Resume => stream.resume_stream(),
+                                        _ => (),
+                                    }
                                 }
-                            }
 
-                            // Propagate the barrier.
-                            yield msg;
+                                // Propagate the barrier.
+                                yield msg;
+                            }
+                            // Only barrier can be received.
+                            _ => unreachable!(),
+                        },
+                        // Chunked FsPage arrives.
+                        Either::Right(chunk) => {
+                            yield Message::Chunk(chunk);
                         }
-                        // Only barrier can be received.
-                        _ => unreachable!(),
                     },
-                    // Chunked FsPage arrives.
-                    Either::Right(chunk) => {
-                        yield Message::Chunk(chunk);
-                    }
-                },
+                }
             }
+
+            stream.replace_data_stream(
+                self.build_chunked_paginate_stream(&source_desc)
+                    .map_err(StreamExecutorError::from)?,
+            );
         }
     }
 }
diff --git a/src/stream/src/executor/source/source_backfill_executor.rs b/src/stream/src/executor/source/source_backfill_executor.rs
index 7135a3fc3bf8e..4a34eabe97e16 100644
--- a/src/stream/src/executor/source/source_backfill_executor.rs
+++ b/src/stream/src/executor/source/source_backfill_executor.rs
@@ -21,7 +21,7 @@ use anyhow::anyhow;
 use either::Either;
 use futures::stream::{select_with_strategy, PollNext};
 use itertools::Itertools;
-use risingwave_common::buffer::BitmapBuilder;
+use risingwave_common::bitmap::BitmapBuilder;
 use risingwave_common::metrics::{LabelGuardedIntCounter, GLOBAL_ERROR_METRICS};
 use risingwave_common::system_param::local_manager::SystemParamsReaderRef;
 use risingwave_common::system_param::reader::SystemParamsRead;
diff --git a/src/stream/src/executor/source/source_executor.rs b/src/stream/src/executor/source/source_executor.rs
index 8e5c3f9726c28..c550c0e12f030 100644
--- a/src/stream/src/executor/source/source_executor.rs
+++ b/src/stream/src/executor/source/source_executor.rs
@@ -909,6 +909,7 @@ mod tests {
                     ],
                 },
                 pause: false,
+                subscriptions_to_add: vec![],
             }));
         barrier_tx.send(init_barrier).unwrap();
 
@@ -998,6 +999,7 @@ mod tests {
                     ],
                 },
                 pause: false,
+                subscriptions_to_add: vec![],
             }));
         barrier_tx.send(init_barrier).unwrap();
 
diff --git a/src/stream/src/executor/source/state_table_handler.rs b/src/stream/src/executor/source/state_table_handler.rs
index 1d14de9a493c9..1c9615c542bfe 100644
--- a/src/stream/src/executor/source/state_table_handler.rs
+++ b/src/stream/src/executor/source/state_table_handler.rs
@@ -28,7 +28,7 @@ use std::ops::{Bound, Deref};
 use std::sync::Arc;
 
 use futures::{pin_mut, StreamExt};
-use risingwave_common::buffer::Bitmap;
+use risingwave_common::bitmap::Bitmap;
 use risingwave_common::hash::VirtualNode;
 use risingwave_common::row::{OwnedRow, Row};
 use risingwave_common::types::{JsonbVal, ScalarImpl, ScalarRef, ScalarRefImpl};
@@ -144,7 +144,7 @@ impl<S: StateStore> SourceStateTableHandler<S> {
     ) -> StreamExecutorResult<()> {
         if states.is_empty() {
             // TODO should be a clear Error Code
-            bail!("states require not null");
+            bail!("states should not be null");
         } else {
             for split in states {
                 self.set_complete(split.id(), split.encode_to_json())
diff --git a/src/stream/src/executor/temporal_join.rs b/src/stream/src/executor/temporal_join.rs
index ce0c4d29621d6..8a994acb5e8db 100644
--- a/src/stream/src/executor/temporal_join.rs
+++ b/src/stream/src/executor/temporal_join.rs
@@ -23,7 +23,7 @@ use itertools::Itertools;
 use local_stats_alloc::{SharedStatsAlloc, StatsAlloc};
 use lru::DefaultHasher;
 use risingwave_common::array::Op;
-use risingwave_common::buffer::BitmapBuilder;
+use risingwave_common::bitmap::BitmapBuilder;
 use risingwave_common::hash::{HashKey, NullBitmap};
 use risingwave_common::row::RowExt;
 use risingwave_common::util::iter_util::ZipEqDebug;
@@ -200,7 +200,7 @@ impl<K: HashKey, S: StateStore> TemporalSide<K, S> {
     }
 }
 
-enum InternalMessage {
+pub(super) enum InternalMessage {
     Chunk(StreamChunk),
     Barrier(Vec<StreamChunk>, Barrier),
     WaterMark(Watermark),
@@ -245,7 +245,7 @@ async fn internal_messages_until_barrier(stream: impl MessageStream, expected_ba
 // any number of `InternalMessage::Chunk(left_chunk)` and followed by
 // `InternalMessage::Barrier(right_chunks, barrier)`.
 #[try_stream(ok = InternalMessage, error = StreamExecutorError)]
-async fn align_input(left: Executor, right: Executor) {
+pub(super) async fn align_input<const YIELD_RIGHT_CHUNKS: bool>(left: Executor, right: Executor) {
     let mut left = pin!(left.execute());
     let mut right = pin!(right.execute());
     // Keep producing intervals until stream exhaustion or errors.
@@ -260,12 +260,18 @@ async fn align_input(left: Executor, right: Executor) {
             );
             match combined.next().await {
                 Some(Either::Left(Ok(Message::Chunk(c)))) => yield InternalMessage::Chunk(c),
-                Some(Either::Right(Ok(Message::Chunk(c)))) => right_chunks.push(c),
+                Some(Either::Right(Ok(Message::Chunk(c)))) => {
+                    if YIELD_RIGHT_CHUNKS {
+                        right_chunks.push(c);
+                    }
+                }
                 Some(Either::Left(Ok(Message::Barrier(b)))) => {
                     let mut remain = chunks_until_barrier(right.by_ref(), b.clone())
                         .try_collect()
                         .await?;
-                    right_chunks.append(&mut remain);
+                    if YIELD_RIGHT_CHUNKS {
+                        right_chunks.append(&mut remain);
+                    }
                     yield InternalMessage::Barrier(right_chunks, b);
                     break 'inner;
                 }
@@ -292,7 +298,18 @@ async fn align_input(left: Executor, right: Executor) {
     }
 }
 
-mod phase1 {
+pub(super) fn apply_indices_map(chunk: StreamChunk, indices: &[usize]) -> StreamChunk {
+    let (data_chunk, ops) = chunk.into_parts();
+    let (columns, vis) = data_chunk.into_parts();
+    let output_columns = indices
+        .iter()
+        .cloned()
+        .map(|idx| columns[idx].clone())
+        .collect();
+    StreamChunk::with_visibility(ops, output_columns, vis)
+}
+
+pub(super) mod phase1 {
     use std::ops::Bound;
 
     use futures::{pin_mut, StreamExt};
@@ -310,7 +327,7 @@ mod phase1 {
     use crate::common::table::state_table::StateTable;
     use crate::executor::monitor::TemporalJoinMetrics;
 
-    pub(super) trait Phase1Evaluation {
+    pub trait Phase1Evaluation {
         /// Called when a matched row is found.
         #[must_use = "consume chunk if produced"]
         fn append_matched_row(
@@ -331,9 +348,9 @@ mod phase1 {
         ) -> Option<StreamChunk>;
     }
 
-    pub(super) struct Inner;
-    pub(super) struct LeftOuter;
-    pub(super) struct LeftOuterWithCond;
+    pub struct Inner;
+    pub struct LeftOuter;
+    pub struct LeftOuterWithCond;
 
     impl Phase1Evaluation for Inner {
         fn append_matched_row(
@@ -635,17 +652,6 @@ impl<K: HashKey, S: StateStore, const T: JoinTypePrimitive, const APPEND_ONLY: b
         }
     }
 
-    fn apply_indices_map(chunk: StreamChunk, indices: &[usize]) -> StreamChunk {
-        let (data_chunk, ops) = chunk.into_parts();
-        let (columns, vis) = data_chunk.into_parts();
-        let output_columns = indices
-            .iter()
-            .cloned()
-            .map(|idx| columns[idx].clone())
-            .collect();
-        StreamChunk::with_visibility(ops, output_columns, vis)
-    }
-
     #[try_stream(ok = Message, error = StreamExecutorError)]
     async fn into_stream(mut self) {
         let right_size = self.right.schema().len();
@@ -682,7 +688,7 @@ impl<K: HashKey, S: StateStore, const T: JoinTypePrimitive, const APPEND_ONLY: b
         let mut wait_first_barrier = true;
 
         #[for_await]
-        for msg in align_input(self.left, self.right) {
+        for msg in align_input::<true>(self.left, self.right) {
             self.right_table.cache.evict();
             self.metrics
                 .temporal_join_cached_entry_count
@@ -725,8 +731,7 @@ impl<K: HashKey, S: StateStore, const T: JoinTypePrimitive, const APPEND_ONLY: b
                             } else {
                                 chunk
                             };
-                            let new_chunk =
-                                Self::apply_indices_map(new_chunk, &self.output_indices);
+                            let new_chunk = apply_indices_map(new_chunk, &self.output_indices);
                             yield Message::Chunk(new_chunk);
                         }
                     } else if let Some(ref cond) = self.condition {
@@ -775,7 +780,7 @@ impl<K: HashKey, S: StateStore, const T: JoinTypePrimitive, const APPEND_ONLY: b
                                 };
                                 new_vis.append(vis);
                             }
-                            let new_chunk = Self::apply_indices_map(
+                            let new_chunk = apply_indices_map(
                                 StreamChunk::with_visibility(ops, columns, new_vis.finish()),
                                 &self.output_indices,
                             );
@@ -800,7 +805,7 @@ impl<K: HashKey, S: StateStore, const T: JoinTypePrimitive, const APPEND_ONLY: b
                         #[for_await]
                         for chunk in st1 {
                             let chunk = chunk?;
-                            let new_chunk = Self::apply_indices_map(chunk, &self.output_indices);
+                            let new_chunk = apply_indices_map(chunk, &self.output_indices);
                             yield Message::Chunk(new_chunk);
                         }
                     }
diff --git a/src/stream/src/executor/test_utils.rs b/src/stream/src/executor/test_utils.rs
index 785384307ecf4..db024411ea0ad 100644
--- a/src/stream/src/executor/test_utils.rs
+++ b/src/stream/src/executor/test_utils.rs
@@ -277,7 +277,7 @@ pub mod agg_executor {
     use risingwave_common::hash::SerializedKey;
     use risingwave_common::types::DataType;
     use risingwave_common::util::sort_util::{ColumnOrder, OrderType};
-    use risingwave_expr::aggregate::{AggCall, AggKind};
+    use risingwave_expr::aggregate::{AggCall, AggKind, PbAggKind};
     use risingwave_pb::stream_plan::PbAggNodeVersion;
     use risingwave_storage::StateStore;
 
@@ -328,7 +328,7 @@ pub mod agg_executor {
         is_append_only: bool,
     ) -> AggStateStorage<S> {
         match agg_call.kind {
-            AggKind::Min | AggKind::Max if !is_append_only => {
+            AggKind::Builtin(PbAggKind::Min | PbAggKind::Max) if !is_append_only => {
                 let input_fields = input_ref.schema().fields();
 
                 let mut column_descs = Vec::new();
@@ -354,7 +354,7 @@ pub mod agg_executor {
                     add_column(*idx, input_fields[*idx].data_type(), None);
                 }
 
-                add_column(agg_call.args.val_indices()[0], agg_call.args.arg_types()[0].clone(), if agg_call.kind == AggKind::Max {
+                add_column(agg_call.args.val_indices()[0], agg_call.args.arg_types()[0].clone(), if matches!(agg_call.kind, AggKind::Builtin(PbAggKind::Max)) {
                     Some(OrderType::descending())
                 } else {
                     Some(OrderType::ascending())
@@ -374,13 +374,15 @@ pub mod agg_executor {
 
                 AggStateStorage::MaterializedInput { table: state_table, mapping: StateTableColumnMapping::new(upstream_columns, None), order_columns }
             }
-            AggKind::Min /* append only */
-            | AggKind::Max /* append only */
-            | AggKind::Sum
-            | AggKind::Sum0
-            | AggKind::Count
-            | AggKind::Avg
-            | AggKind::ApproxCountDistinct => {
+            AggKind::Builtin(
+                PbAggKind::Min /* append only */
+                | PbAggKind::Max /* append only */
+                | PbAggKind::Sum
+                | PbAggKind::Sum0
+                | PbAggKind::Count
+                | PbAggKind::Avg
+                | PbAggKind::ApproxCountDistinct
+            ) => {
                 AggStateStorage::Value
             }
             _ => {
diff --git a/src/stream/src/executor/top_n/group_top_n.rs b/src/stream/src/executor/top_n/group_top_n.rs
index 9457e2f7729c6..0b65dcaa46829 100644
--- a/src/stream/src/executor/top_n/group_top_n.rs
+++ b/src/stream/src/executor/top_n/group_top_n.rs
@@ -15,7 +15,7 @@
 use std::ops::{Deref, DerefMut};
 
 use risingwave_common::array::Op;
-use risingwave_common::buffer::Bitmap;
+use risingwave_common::bitmap::Bitmap;
 use risingwave_common::hash::HashKey;
 use risingwave_common::row::RowExt;
 use risingwave_common::util::epoch::EpochPair;
@@ -239,8 +239,7 @@ where
 
     async fn handle_watermark(&mut self, watermark: Watermark) -> Option<Watermark> {
         if watermark.col_idx == self.group_by[0] {
-            self.managed_state
-                .update_watermark(watermark.val.clone(), false);
+            self.managed_state.update_watermark(watermark.val.clone());
             Some(watermark)
         } else {
             None
diff --git a/src/stream/src/executor/top_n/group_top_n_appendonly.rs b/src/stream/src/executor/top_n/group_top_n_appendonly.rs
index b8e4094486624..2f9bd2d547025 100644
--- a/src/stream/src/executor/top_n/group_top_n_appendonly.rs
+++ b/src/stream/src/executor/top_n/group_top_n_appendonly.rs
@@ -13,7 +13,7 @@
 // limitations under the License.
 
 use risingwave_common::array::Op;
-use risingwave_common::buffer::Bitmap;
+use risingwave_common::bitmap::Bitmap;
 use risingwave_common::hash::HashKey;
 use risingwave_common::row::{RowDeserializer, RowExt};
 use risingwave_common::util::epoch::EpochPair;
@@ -208,8 +208,7 @@ where
 
     async fn handle_watermark(&mut self, watermark: Watermark) -> Option<Watermark> {
         if watermark.col_idx == self.group_by[0] {
-            self.managed_state
-                .update_watermark(watermark.val.clone(), false);
+            self.managed_state.update_watermark(watermark.val.clone());
             Some(watermark)
         } else {
             None
diff --git a/src/stream/src/executor/top_n/mod.rs b/src/stream/src/executor/top_n/mod.rs
index b33f5f4883cac..2ad3c0ed7330e 100644
--- a/src/stream/src/executor/top_n/mod.rs
+++ b/src/stream/src/executor/top_n/mod.rs
@@ -19,9 +19,7 @@ use utils::*;
 mod top_n_cache;
 mod top_n_state;
 use top_n_cache::{TopNCache, TopNCacheTrait};
-mod topn_cache_state;
 use top_n_state::ManagedTopNState;
-use topn_cache_state::CacheKey;
 
 // `TopN` variants
 mod group_top_n;
diff --git a/src/stream/src/executor/top_n/top_n_cache.rs b/src/stream/src/executor/top_n/top_n_cache.rs
index 3ce58e4e6017d..d8211b4ad076c 100644
--- a/src/stream/src/executor/top_n/top_n_cache.rs
+++ b/src/stream/src/executor/top_n/top_n_cache.rs
@@ -20,14 +20,18 @@ use itertools::Itertools;
 use risingwave_common::array::{Op, RowRef};
 use risingwave_common::row::{CompactedRow, Row, RowDeserializer, RowExt};
 use risingwave_common::types::DataType;
+use risingwave_common_estimate_size::collections::EstimatedBTreeMap;
 use risingwave_common_estimate_size::EstimateSize;
 use risingwave_storage::StateStore;
 
-use super::topn_cache_state::TopNCacheState;
-use super::{CacheKey, GroupKey, ManagedTopNState};
+use super::{GroupKey, ManagedTopNState};
 use crate::consistency::{consistency_error, enable_strict_consistency};
 use crate::executor::error::StreamExecutorResult;
 
+/// `CacheKey` is composed of `(order_by, remaining columns of pk)`.
+pub type CacheKey = (Vec<u8>, Vec<u8>);
+pub type Cache = EstimatedBTreeMap<CacheKey, CompactedRow>;
+
 const TOPN_CACHE_HIGH_CAPACITY_FACTOR: usize = 2;
 const TOPN_CACHE_MIN_CAPACITY: usize = 10;
 
@@ -43,19 +47,32 @@ const TOPN_CACHE_MIN_CAPACITY: usize = 10;
 /// `OFFSET m FETCH FIRST n ROWS WITH TIES` and `m <= RANK() <= n` are not supported now,
 /// since they have different semantics.
 pub struct TopNCache<const WITH_TIES: bool> {
-    /// Rows in the range `[0, offset)`
-    pub low: TopNCacheState,
-    /// Rows in the range `[offset, offset+limit)`
+    /// Rows in the range `[0, offset)`. Should always be synced with state table.
+    pub low: Option<Cache>,
+
+    /// Rows in the range `[offset, offset+limit)`. Should always be synced with state table.
     ///
     /// When `WITH_TIES` is true, it also stores ties for the last element,
     /// and thus the size can be larger than `limit`.
-    pub middle: TopNCacheState,
-    /// Rows in the range `[offset+limit, offset+limit+high_capacity)`
+    pub middle: Cache,
+
+    /// Cache of the beginning rows in the range `[offset+limit, ...)`.
     ///
-    /// When `WITH_TIES` is true, it also stores ties for the last element,
-    /// and thus the size can be larger than `high_capacity`.
-    pub high: TopNCacheState,
-    pub high_capacity: usize,
+    /// This is very similar to [`TopNStateCache`], which only caches the top-N rows in the table
+    /// and only accepts new records that are less than the largest in the cache.
+    ///
+    /// When `WITH_TIES` is true, it guarantees that the ties of the last element are in the cache,
+    /// and thus the size can be larger than `rest_cache_capacity`.
+    ///
+    /// When the cache becomes empty, if the `table_row_count` is not matched, we need to view the cache
+    /// as unsynced and refill it from the state table.
+    ///
+    /// TODO(rc): later we should reuse [`TopNStateCache`] here.
+    ///
+    /// [`TopNStateCache`]: crate::common::state_cache::TopNStateCache
+    pub high: Cache,
+    pub high_cache_capacity: usize,
+
     pub offset: usize,
     /// Assumption: `limit != 0`
     pub limit: usize,
@@ -83,13 +100,13 @@ impl<const WITH_TIES: bool> Debug for TopNCache<WITH_TIES> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         write!(
             f,
-            "TopNCache {{\n  offset: {}, limit: {}, high_capacity: {},\n",
-            self.offset, self.limit, self.high_capacity
+            "TopNCache {{\n  offset: {}, limit: {}, high_cache_capacity: {},\n",
+            self.offset, self.limit, self.high_cache_capacity
         )?;
 
         fn format_cache(
             f: &mut std::fmt::Formatter<'_>,
-            cache: &TopNCacheState,
+            cache: &Cache,
             data_types: &[DataType],
         ) -> std::fmt::Result {
             if cache.is_empty() {
@@ -109,7 +126,11 @@ impl<const WITH_TIES: bool> Debug for TopNCache<WITH_TIES> {
         }
 
         writeln!(f, "  low:")?;
-        format_cache(f, &self.low, &self.data_types)?;
+        if let Some(low) = &self.low {
+            format_cache(f, low, &self.data_types)?;
+        } else {
+            writeln!(f, "    <none>")?;
+        }
         writeln!(f, "\n  middle:")?;
         format_cache(f, &self.middle, &self.data_types)?;
         writeln!(f, "\n  high:")?;
@@ -142,7 +163,7 @@ pub trait TopNCacheTrait {
     /// Changes in `self.middle` is recorded to `res_ops` and `res_rows`, which will be
     /// used to generate messages to be sent to downstream operators.
     ///
-    /// Because we may need to add data from the state table to `self.high` during the delete
+    /// Because we may need to refill data from the state table to `self.high` during the delete
     /// operation, we need to pass in `group_key`, `epoch` and `managed_state` to do a prefix
     /// scan of the state table.
     #[allow(clippy::too_many_arguments)]
@@ -160,21 +181,22 @@ pub trait TopNCacheTrait {
 impl<const WITH_TIES: bool> TopNCache<WITH_TIES> {
     /// `data_types` -- Data types for the full row.
     pub fn new(offset: usize, limit: usize, data_types: Vec<DataType>) -> Self {
-        assert!(limit != 0);
+        assert!(limit > 0);
         if WITH_TIES {
             // It's trickier to support.
             // Also `OFFSET WITH TIES` has different semantic with `a < RANK() < b`
             assert!(offset == 0, "OFFSET is not supported with WITH TIES");
         }
+        let high_cache_capacity = offset
+            .checked_add(limit)
+            .and_then(|v| v.checked_mul(TOPN_CACHE_HIGH_CAPACITY_FACTOR))
+            .unwrap_or(usize::MAX)
+            .max(TOPN_CACHE_MIN_CAPACITY);
         Self {
-            low: TopNCacheState::new(),
-            middle: TopNCacheState::new(),
-            high: TopNCacheState::new(),
-            high_capacity: offset
-                .checked_add(limit)
-                .and_then(|v| v.checked_mul(TOPN_CACHE_HIGH_CAPACITY_FACTOR))
-                .unwrap_or(usize::MAX)
-                .max(TOPN_CACHE_MIN_CAPACITY),
+            low: if offset > 0 { Some(Cache::new()) } else { None },
+            middle: Cache::new(),
+            high: Cache::new(),
+            high_cache_capacity,
             offset,
             limit,
             table_row_count: None,
@@ -185,37 +207,35 @@ impl<const WITH_TIES: bool> TopNCache<WITH_TIES> {
     /// Clear the cache. After this, the cache must be `init` again before use.
     #[allow(dead_code)]
     pub fn clear(&mut self) {
-        self.low.clear();
+        self.low.as_mut().map(Cache::clear);
         self.middle.clear();
         self.high.clear();
     }
 
     /// Get total count of entries in the cache.
     pub fn len(&self) -> usize {
-        self.low.len() + self.middle.len() + self.high.len()
+        self.low.as_ref().map(Cache::len).unwrap_or(0) + self.middle.len() + self.high.len()
     }
 
     pub(super) fn update_table_row_count(&mut self, table_row_count: usize) {
         self.table_row_count = Some(table_row_count)
     }
 
-    fn table_row_count_matched(&self) -> bool {
-        self.table_row_count
-            .map(|n| n == self.len())
-            .unwrap_or(false)
-    }
-
-    pub fn is_low_cache_full(&self) -> bool {
-        assert!(self.low.len() <= self.offset);
-        let full = self.low.len() == self.offset;
-        if !full {
-            assert!(self.middle.is_empty());
-            assert!(self.high.is_empty());
+    pub fn low_is_full(&self) -> bool {
+        if let Some(low) = &self.low {
+            assert!(low.len() <= self.offset);
+            let full = low.len() == self.offset;
+            if !full {
+                assert!(self.middle.is_empty());
+                assert!(self.high.is_empty());
+            }
+            full
+        } else {
+            true
         }
-        full
     }
 
-    pub fn is_middle_cache_full(&self) -> bool {
+    pub fn middle_is_full(&self) -> bool {
         // For WITH_TIES, the middle cache can exceed the capacity.
         if !WITH_TIES {
             assert!(
@@ -225,7 +245,7 @@ impl<const WITH_TIES: bool> TopNCache<WITH_TIES> {
         }
         let full = self.middle.len() >= self.limit;
         if full {
-            assert!(self.is_low_cache_full());
+            assert!(self.low_is_full());
         } else {
             assert!(
                 self.high.is_empty(),
@@ -235,46 +255,34 @@ impl<const WITH_TIES: bool> TopNCache<WITH_TIES> {
         full
     }
 
-    pub fn is_high_cache_full(&self) -> bool {
+    pub fn high_is_full(&self) -> bool {
         // For WITH_TIES, the high cache can exceed the capacity.
         if !WITH_TIES {
-            assert!(self.high.len() <= self.high_capacity);
+            assert!(self.high.len() <= self.high_cache_capacity);
         }
-        self.high.len() >= self.high_capacity
+        self.high.len() >= self.high_cache_capacity
     }
 
-    fn last_cache_key_before_high(&self) -> Option<&CacheKey> {
-        let middle_last_key = self.middle.last_key_value().map(|(k, _)| k);
-        middle_last_key.or_else(|| self.low.last_key_value().map(|(k, _)| k))
-    }
-
-    /// Use this method instead of `self.high.insert` directly when possible.
-    ///
-    /// It only inserts into high cache if the key is smaller than the largest key in the high
-    /// cache. Otherwise, we simply ignore the row. We will wait until the high cache becomes
-    /// empty and fill it at that time.
-    fn insert_high_cache(&mut self, cache_key: CacheKey, row: CompactedRow, is_from_middle: bool) {
-        if !self.is_high_cache_full() {
-            if is_from_middle {
-                self.high.insert(cache_key, row);
-                return;
-            }
-            // For direct insert, we need to check if the key is smaller than the largest key
-            if let Some(high_last) = self.high.last_key_value()
-                && cache_key <= *high_last.0
-            {
-                debug_assert!(cache_key != *high_last.0, "cache_key should be unique");
-                self.high.insert(cache_key, row);
-            }
+    fn high_is_synced(&self) -> bool {
+        if !self.high.is_empty() {
+            true
         } else {
-            let high_last = self.high.last_entry().unwrap();
-            if cache_key <= *high_last.key() {
-                debug_assert!(cache_key != *high_last.key(), "cache_key should be unique");
-                high_last.remove_entry();
-                self.high.insert(cache_key, row);
-            }
+            // check if table row count matches
+            self.table_row_count
+                .map(|n| n == self.len())
+                .unwrap_or(false)
         }
     }
+
+    fn last_cache_key_before_high(&self) -> Option<&CacheKey> {
+        let middle_last_key = self.middle.last_key_value().map(|(k, _)| k);
+        middle_last_key.or_else(|| {
+            self.low
+                .as_ref()
+                .and_then(Cache::last_key_value)
+                .map(|(k, _)| k)
+        })
+    }
 }
 
 impl TopNCacheTrait for TopNCache<false> {
@@ -289,59 +297,85 @@ impl TopNCacheTrait for TopNCache<false> {
             *row_count += 1;
         }
 
-        if !self.is_low_cache_full() {
-            self.low.insert(cache_key, (&row).into());
-            return;
-        }
-        let elem_to_compare_with_middle = if let Some(low_last) = self.low.last_entry()
-            && cache_key <= *low_last.key()
-        {
-            // Take the last element of `cache.low` and insert input row to it.
-            let low_last = low_last.remove_entry();
-            self.low.insert(cache_key, (&row).into());
-            low_last
-        } else {
-            (cache_key, (&row).into())
+        let mut append_res = |op: Op, row: CompactedRow| {
+            res_ops.push(op);
+            res_rows.push(row);
         };
 
-        if !self.is_middle_cache_full() {
-            self.middle.insert(
-                elem_to_compare_with_middle.0,
-                elem_to_compare_with_middle.1.clone(),
-            );
-            res_ops.push(Op::Insert);
-            res_rows.push(elem_to_compare_with_middle.1);
+        let mut to_insert = (cache_key, (&row).into());
+        let mut is_last_of_lower_cache = false; // for saving one key comparison
+
+        let low_is_full = self.low_is_full();
+        if let Some(low) = &mut self.low {
+            // try insert into low cache
+
+            if !low_is_full {
+                low.insert(to_insert.0, to_insert.1);
+                return;
+            }
+
+            // low cache is full
+            let low_last = low.last_entry().unwrap();
+            if &to_insert.0 < low_last.key() {
+                // make space for the new entry
+                let low_last = low_last.remove_entry();
+                low.insert(to_insert.0, to_insert.1);
+                to_insert = low_last; // move the last entry to the middle cache
+                is_last_of_lower_cache = true;
+            }
+        }
+
+        // try insert into middle cache
+
+        if !self.middle_is_full() {
+            self.middle.insert(to_insert.0, to_insert.1.clone());
+            append_res(Op::Insert, to_insert.1);
             return;
         }
 
-        let mut is_from_middle = false;
-        let elem_to_compare_with_high = {
-            let middle_last = self.middle.last_entry().unwrap();
-            if elem_to_compare_with_middle.0 <= *middle_last.key() {
-                // If the row in the range of [offset, offset+limit), the largest row in
-                // `cache.middle` needs to be moved to `cache.high`
-                let res = middle_last.remove_entry();
-                res_ops.push(Op::Delete);
-                res_rows.push(res.1.clone());
-                res_ops.push(Op::Insert);
-                res_rows.push(elem_to_compare_with_middle.1.clone());
-                self.middle
-                    .insert(elem_to_compare_with_middle.0, elem_to_compare_with_middle.1);
-                is_from_middle = true;
-                res
-            } else {
-                elem_to_compare_with_middle
+        // middle cache is full
+        let middle_last = self.middle.last_entry().unwrap();
+        if is_last_of_lower_cache || &to_insert.0 < middle_last.key() {
+            // make space for the new entry
+            let middle_last = middle_last.remove_entry();
+            self.middle.insert(to_insert.0, to_insert.1.clone());
+
+            append_res(Op::Delete, middle_last.1.clone());
+            append_res(Op::Insert, to_insert.1);
+
+            to_insert = middle_last; // move the last entry to the high cache
+            is_last_of_lower_cache = true;
+        }
+
+        // try insert into high cache
+
+        // The logic is a bit different from the other two caches, because high cache is not
+        // guaranteed to be fully synced with the "high part" of the table.
+
+        if is_last_of_lower_cache || self.high_is_synced() {
+            // For `is_last_of_lower_cache`, an obvious observation is that the key to insert is
+            // always smaller than any key in the high part of the table.
+
+            if self.high.is_empty() {
+                // if high cache is empty, we can insert directly
+                self.high.insert(to_insert.0, to_insert.1);
+                return;
             }
-        };
 
-        self.insert_high_cache(
-            elem_to_compare_with_high.0,
-            elem_to_compare_with_high.1,
-            is_from_middle,
-        );
+            let high_is_full = self.high_is_full();
+            let high_last = self.high.last_entry().unwrap();
+
+            if is_last_of_lower_cache || &to_insert.0 < high_last.key() {
+                // we can only insert if the key is smaller than the largest key in the high cache
+                if high_is_full {
+                    // make space for the new entry
+                    high_last.remove_entry();
+                }
+                self.high.insert(to_insert.0, to_insert.1);
+            }
+        }
     }
 
-    #[allow(clippy::too_many_arguments)]
     async fn delete<S: StateStore>(
         &mut self,
         group_key: Option<impl GroupKey>,
@@ -361,66 +395,94 @@ impl TopNCacheTrait for TopNCache<false> {
             *row_count -= 1;
         }
 
-        if self.is_middle_cache_full() && cache_key > *self.middle.last_key_value().unwrap().0 {
-            // The row is in high
+        let mut append_res = |op: Op, row: CompactedRow| {
+            res_ops.push(op);
+            res_rows.push(row);
+        };
+
+        if self.middle_is_full() && &cache_key > self.middle.last_key_value().unwrap().0 {
+            // the row is in high
             self.high.remove(&cache_key);
-        } else if self.is_low_cache_full()
-            && (self.offset == 0 || cache_key > *self.low.last_key_value().unwrap().0)
+        } else if self.low_is_full()
+            && self
+                .low
+                .as_ref()
+                .map(|low| &cache_key > low.last_key_value().unwrap().0)
+                .unwrap_or(
+                    true, // if low is None, `cache_key` should be in middle
+                )
         {
-            // The row is in mid
-            self.middle.remove(&cache_key);
-            res_ops.push(Op::Delete);
-            res_rows.push((&row).into());
+            // the row is in middle
+            let removed = self.middle.remove(&cache_key);
+            append_res(Op::Delete, (&row).into());
+
+            if removed.is_none() {
+                // the middle cache should always be synced, if the key is not found, then it also doesn't
+                // exist in the state table
+                consistency_error!(
+                    ?group_key,
+                    ?cache_key,
+                    "cache key not found in middle cache"
+                );
+                return Ok(());
+            }
 
-            // Try to fill the high cache if it is empty
-            if self.high.is_empty() && !self.table_row_count_matched() {
+            // refill the high cache if it's not synced
+            if !self.high_is_synced() {
+                self.high.clear();
                 managed_state
                     .fill_high_cache(
                         group_key,
                         self,
                         self.last_cache_key_before_high().cloned(),
-                        self.high_capacity,
+                        self.high_cache_capacity,
                     )
                     .await?;
             }
 
-            // Bring one element, if any, from high cache to middle cache
+            // bring one element, if any, from high cache to middle cache
             if !self.high.is_empty() {
                 let high_first = self.high.pop_first().unwrap();
-                res_ops.push(Op::Insert);
-                res_rows.push(high_first.1.clone());
+                append_res(Op::Insert, high_first.1.clone());
                 self.middle.insert(high_first.0, high_first.1);
             }
 
             assert!(self.high.is_empty() || self.middle.len() == self.limit);
         } else {
-            // The row is in low
-            self.low.remove(&cache_key);
+            // the row is in low
+            let low = self.low.as_mut().unwrap();
+            let removed = low.remove(&cache_key);
+
+            if removed.is_none() {
+                // the low cache should always be synced, if the key is not found, then it also doesn't
+                // exist in the state table
+                consistency_error!(?group_key, ?cache_key, "cache key not found in low cache");
+                return Ok(());
+            }
 
-            // Bring one element, if any, from middle cache to low cache
+            // bring one element, if any, from middle cache to low cache
             if !self.middle.is_empty() {
                 let middle_first = self.middle.pop_first().unwrap();
-                res_ops.push(Op::Delete);
-                res_rows.push(middle_first.1.clone());
-                self.low.insert(middle_first.0, middle_first.1);
+                append_res(Op::Delete, middle_first.1.clone());
+                low.insert(middle_first.0, middle_first.1);
 
-                // Try to fill the high cache if it is empty
-                if self.high.is_empty() && !self.table_row_count_matched() {
+                // fill the high cache if it's not synced
+                if !self.high_is_synced() {
+                    self.high.clear();
                     managed_state
                         .fill_high_cache(
                             group_key,
                             self,
                             self.last_cache_key_before_high().cloned(),
-                            self.high_capacity,
+                            self.high_cache_capacity,
                         )
                         .await?;
                 }
 
-                // Bring one element, if any, from high cache to middle cache
+                // bring one element, if any, from high cache to middle cache
                 if !self.high.is_empty() {
                     let high_first = self.high.pop_first().unwrap();
-                    res_ops.push(Op::Insert);
-                    res_rows.push(high_first.1.clone());
+                    append_res(Op::Insert, high_first.1.clone());
                     self.middle.insert(high_first.0, high_first.1);
                 }
             }
@@ -443,32 +505,36 @@ impl TopNCacheTrait for TopNCache<true> {
         }
 
         assert!(
-            self.low.is_empty(),
-            "Offset is not supported yet for WITH TIES, so low cache should be empty"
+            self.low.is_none(),
+            "Offset is not supported yet for WITH TIES, so low cache should be None"
         );
 
-        let elem_to_compare_with_middle = (cache_key, row);
+        let mut append_res = |op: Op, row: CompactedRow| {
+            res_ops.push(op);
+            res_rows.push(row);
+        };
 
-        if !self.is_middle_cache_full() {
-            self.middle.insert(
-                elem_to_compare_with_middle.0.clone(),
-                (&elem_to_compare_with_middle.1).into(),
-            );
-            res_ops.push(Op::Insert);
-            res_rows.push((&elem_to_compare_with_middle.1).into());
+        let to_insert: (CacheKey, CompactedRow) = (cache_key, (&row).into());
+
+        // try insert into middle cache
+
+        if !self.middle_is_full() {
+            self.middle.insert(to_insert.0.clone(), to_insert.1.clone());
+            append_res(Op::Insert, to_insert.1);
             return;
         }
 
-        let sort_key = &elem_to_compare_with_middle.0 .0;
-        let middle_last = self.middle.last_key_value().unwrap();
-        let middle_last_order_by = &middle_last.0 .0.clone();
+        // middle cache is full
 
-        match sort_key.cmp(middle_last_order_by) {
+        let to_insert_sort_key = &(to_insert.0).0;
+        let middle_last_sort_key = self.middle.last_key().unwrap().0.clone();
+
+        match to_insert_sort_key.cmp(&middle_last_sort_key) {
             Ordering::Less => {
-                // The row is in middle.
-                let num_ties = self
+                // the row is in middle
+                let n_ties_of_last = self
                     .middle
-                    .range((middle_last_order_by.clone(), vec![])..)
+                    .range((middle_last_sort_key.clone(), vec![])..)
                     .count();
                 // We evict the last row and its ties only if the number of remaining rows still is
                 // still larger than limit, i.e., there are limit-1 other rows.
@@ -477,51 +543,67 @@ impl TopNCacheTrait for TopNCache<true> {
                 // insert 0 -> [0,1,1,1,1]
                 // insert 0 -> [0,0,1,1,1,1]
                 // insert 0 -> [0,0,0]
-                if self.middle.len() - num_ties + 1 >= self.limit {
+                if self.middle.len() + 1 - n_ties_of_last >= self.limit {
+                    // Middle will be full without the last element and its ties after insertion.
+                    // Let's move the last element and its ties to high cache first.
                     while let Some(middle_last) = self.middle.last_entry()
-                        && middle_last.key().0 == middle_last_order_by.clone()
+                        && middle_last.key().0 == middle_last_sort_key
                     {
                         let middle_last = middle_last.remove_entry();
-                        res_ops.push(Op::Delete);
-                        res_rows.push(middle_last.1.clone());
+                        append_res(Op::Delete, middle_last.1.clone());
+                        // we can blindly move entries from middle cache to high cache no matter high cache is synced or not
                         self.high.insert(middle_last.0, middle_last.1);
                     }
                 }
-                if self.high.len() >= self.high_capacity {
+                if self.high.len() > self.high_cache_capacity {
+                    // evict some entries from high cache if it exceeds the capacity
                     let high_last = self.high.pop_last().unwrap();
-                    let high_last_order_by = high_last.0 .0;
-                    self.high.retain(|k, _| k.0 != high_last_order_by);
+                    let high_last_sort_key = (high_last.0).0;
+                    // Remove all ties of the last element in high cache, for the sake of simplicity.
+                    // This may cause repeatedly refill the high cache if number of ties is large.
+                    self.high.retain(|k, _| k.0 != high_last_sort_key);
                 }
 
-                res_ops.push(Op::Insert);
-                res_rows.push((&elem_to_compare_with_middle.1).into());
-                self.middle.insert(
-                    elem_to_compare_with_middle.0,
-                    (&elem_to_compare_with_middle.1).into(),
-                );
+                append_res(Op::Insert, to_insert.1.clone());
+                self.middle.insert(to_insert.0, to_insert.1);
             }
             Ordering::Equal => {
-                // The row is in middle and is a tie with the last row.
-                res_ops.push(Op::Insert);
-                res_rows.push((&elem_to_compare_with_middle.1).into());
-                self.middle.insert(
-                    elem_to_compare_with_middle.0,
-                    (&elem_to_compare_with_middle.1).into(),
-                );
+                // the row is in middle and is a tie of the last row
+                append_res(Op::Insert, to_insert.1.clone());
+                self.middle.insert(to_insert.0, to_insert.1);
             }
             Ordering::Greater => {
-                // The row is in high.
-                let elem_to_compare_with_high = elem_to_compare_with_middle;
-                self.insert_high_cache(
-                    elem_to_compare_with_high.0,
-                    elem_to_compare_with_high.1.into(),
-                    false,
-                );
+                // the row is in high
+
+                if self.high_is_synced() {
+                    // only insert into high cache if it is synced
+
+                    if self.high.is_empty() {
+                        // if high cache is empty, we can insert directly
+                        self.high.insert(to_insert.0, to_insert.1);
+                        return;
+                    }
+
+                    if to_insert_sort_key <= &self.high.last_key().unwrap().0 {
+                        // We can only insert if the key is <= the largest key in the high cache.
+                        // Note that we have all ties of the last element in the high cache, so we can
+                        // safely compare only the sort key.
+                        self.high.insert(to_insert.0, to_insert.1);
+                    }
+
+                    if self.high.len() > self.high_cache_capacity {
+                        // evict some entries from high cache if it exceeds the capacity
+                        let high_last = self.high.pop_last().unwrap();
+                        let high_last_sort_key = (high_last.0).0;
+                        // Remove all ties of the last element in high cache, for the sake of simplicity.
+                        // This may cause repeatedly refill the high cache if number of ties is large.
+                        self.high.retain(|k, _| k.0 != high_last_sort_key);
+                    }
+                }
             }
         }
     }
 
-    #[allow(clippy::too_many_arguments)]
     async fn delete<S: StateStore>(
         &mut self,
         group_key: Option<impl GroupKey>,
@@ -540,56 +622,65 @@ impl TopNCacheTrait for TopNCache<true> {
             *row_count -= 1;
         }
 
-        // Since low cache is always empty for WITH_TIES, this unwrap is safe.
-        let middle_last = self.middle.last_key_value().unwrap();
-        let middle_last_order_by = middle_last.0 .0.clone();
+        assert!(
+            self.low.is_none(),
+            "Offset is not supported yet for WITH TIES, so low cache should be None"
+        );
 
-        let sort_key = cache_key.0.clone();
-        if sort_key > middle_last_order_by {
-            // The row is in high.
+        let mut append_res = |op: Op, row: CompactedRow| {
+            res_ops.push(op);
+            res_rows.push(row);
+        };
+
+        if self.middle.is_empty() {
+            consistency_error!(
+                ?group_key,
+                ?cache_key,
+                "middle cache is empty, but we receive a DELETE operation"
+            );
+            append_res(Op::Delete, (&row).into());
+            return Ok(());
+        }
+
+        let middle_last_sort_key = self.middle.last_key().unwrap().0.clone();
+
+        let to_delete_sort_key = cache_key.0.clone();
+        if to_delete_sort_key > middle_last_sort_key {
+            // the row is in high
             self.high.remove(&cache_key);
         } else {
-            // The row is in middle
+            // the row is in middle
             self.middle.remove(&cache_key);
-            res_ops.push(Op::Delete);
-            res_rows.push((&row).into());
+            append_res(Op::Delete, (&row).into());
             if self.middle.len() >= self.limit {
-                // This can happen when there are ties.
+                // this can happen when there are ties
                 return Ok(());
             }
 
-            // Try to fill the high cache if it is empty
-            if self.high.is_empty() && !self.table_row_count_matched() {
+            // refill the high cache if it's not synced
+            if !self.high_is_synced() {
                 managed_state
                     .fill_high_cache(
                         group_key,
                         self,
                         self.last_cache_key_before_high().cloned(),
-                        self.high_capacity,
+                        self.high_cache_capacity,
                     )
                     .await?;
             }
 
-            // Bring elements with the same sort key, if any, from high cache to middle cache.
+            // bring the first element and its ties, if any, from high cache to middle cache
             if !self.high.is_empty() {
                 let high_first = self.high.pop_first().unwrap();
-                let high_first_order_by = high_first.0 .0.clone();
-                assert!(high_first_order_by > middle_last_order_by);
+                let high_first_sort_key = (high_first.0).0.clone();
+                assert!(high_first_sort_key > middle_last_sort_key);
 
-                res_ops.push(Op::Insert);
-                res_rows.push(high_first.1.clone());
+                append_res(Op::Insert, high_first.1.clone());
                 self.middle.insert(high_first.0, high_first.1);
 
-                // We need to trigger insert for all rows with prefix `high_first_order_by`
-                // in high cache.
-                for (ordered_pk_row, row) in self.high.extract_if(|k, _| k.0 == high_first_order_by)
-                {
-                    if ordered_pk_row.0 != high_first_order_by {
-                        break;
-                    }
-                    res_ops.push(Op::Insert);
-                    res_rows.push(row.clone());
-                    self.middle.insert(ordered_pk_row, row);
+                for (cache_key, row) in self.high.extract_if(|k, _| k.0 == high_first_sort_key) {
+                    append_res(Op::Insert, row.clone());
+                    self.middle.insert(cache_key, row);
                 }
             }
         }
@@ -628,51 +719,56 @@ impl AppendOnlyTopNCacheTrait for TopNCache<false> {
         managed_state: &mut ManagedTopNState<S>,
         row_deserializer: &RowDeserializer,
     ) -> StreamExecutorResult<()> {
-        if self.is_middle_cache_full() && &cache_key >= self.middle.last_key_value().unwrap().0 {
+        if self.middle_is_full() && &cache_key >= self.middle.last_key().unwrap() {
             return Ok(());
         }
         managed_state.insert(row_ref);
 
-        // Then insert input row to corresponding cache range according to its order key
-        if !self.is_low_cache_full() {
-            self.low.insert(cache_key, row_ref.into());
-            return Ok(());
+        let mut append_res = |op: Op, row: CompactedRow| {
+            res_ops.push(op);
+            res_rows.push(row);
+        };
+
+        // insert input row into corresponding cache according to its sort key
+        let mut to_insert = (cache_key, row_ref.into());
+
+        let low_is_full = self.low_is_full();
+        if let Some(low) = &mut self.low {
+            // try insert into low cache
+
+            if !low_is_full {
+                low.insert(to_insert.0, to_insert.1);
+                return Ok(());
+            }
+
+            // low cache is full
+            let low_last = low.last_entry().unwrap();
+            if &to_insert.0 < low_last.key() {
+                // make space for the new entry
+                let low_last = low_last.remove_entry();
+                low.insert(to_insert.0, to_insert.1);
+                to_insert = low_last; // move the last entry to the middle cache
+            }
         }
 
-        let elem_to_insert_into_middle = if let Some(low_last) = self.low.last_entry()
-            && &cache_key <= low_last.key()
-        {
-            // Take the last element of `cache.low` and insert input row to it.
-            let low_last = low_last.remove_entry();
-            self.low.insert(cache_key, row_ref.into());
-            low_last
-        } else {
-            (cache_key, row_ref.into())
-        };
+        // try insert into middle cache
 
-        if !self.is_middle_cache_full() {
-            self.middle.insert(
-                elem_to_insert_into_middle.0,
-                elem_to_insert_into_middle.1.clone(),
-            );
-            res_ops.push(Op::Insert);
-            res_rows.push(elem_to_insert_into_middle.1);
+        if !self.middle_is_full() {
+            self.middle.insert(to_insert.0, to_insert.1.clone());
+            append_res(Op::Insert, to_insert.1);
             return Ok(());
         }
 
         // The row must be in the range of [offset, offset+limit).
         // the largest row in `cache.middle` needs to be removed.
         let middle_last = self.middle.pop_last().unwrap();
-        debug_assert!(elem_to_insert_into_middle.0 < middle_last.0);
+        debug_assert!(to_insert.0 < middle_last.0);
 
-        res_ops.push(Op::Delete);
-        res_rows.push(middle_last.1.clone());
+        append_res(Op::Delete, middle_last.1.clone());
         managed_state.delete(row_deserializer.deserialize(middle_last.1.row.as_ref())?);
 
-        res_ops.push(Op::Insert);
-        res_rows.push(elem_to_insert_into_middle.1.clone());
-        self.middle
-            .insert(elem_to_insert_into_middle.0, elem_to_insert_into_middle.1);
+        append_res(Op::Insert, to_insert.1.clone());
+        self.middle.insert(to_insert.0, to_insert.1);
 
         // Unlike normal topN, append only topN does not use the high part of the cache.
 
@@ -691,31 +787,38 @@ impl AppendOnlyTopNCacheTrait for TopNCache<true> {
         row_deserializer: &RowDeserializer,
     ) -> StreamExecutorResult<()> {
         assert!(
-            self.low.is_empty(),
+            self.low.is_none(),
             "Offset is not supported yet for WITH TIES, so low cache should be empty"
         );
-        let elem_to_compare_with_middle = (cache_key, row_ref);
-
-        if !self.is_middle_cache_full() {
-            let row: CompactedRow = elem_to_compare_with_middle.1.into();
-            managed_state.insert(elem_to_compare_with_middle.1);
-            self.middle
-                .insert(elem_to_compare_with_middle.0.clone(), row.clone());
-            res_ops.push(Op::Insert);
+
+        let mut append_res = |op: Op, row: CompactedRow| {
+            res_ops.push(op);
             res_rows.push(row);
+        };
+
+        let to_insert = (cache_key, row_ref);
+
+        // try insert into middle cache
+
+        if !self.middle_is_full() {
+            managed_state.insert(to_insert.1);
+            let row: CompactedRow = to_insert.1.into();
+            self.middle.insert(to_insert.0, row.clone());
+            append_res(Op::Insert, row);
             return Ok(());
         }
 
-        let sort_key = &elem_to_compare_with_middle.0 .0;
-        let middle_last = self.middle.last_key_value().unwrap();
-        let middle_last_order_by = &middle_last.0 .0.clone();
+        // middle cache is full
+
+        let to_insert_sort_key = &(to_insert.0).0;
+        let middle_last_sort_key = self.middle.last_key().unwrap().0.clone();
 
-        match sort_key.cmp(middle_last_order_by) {
+        match to_insert_sort_key.cmp(&middle_last_sort_key) {
             Ordering::Less => {
-                // The row is in middle.
-                let num_ties = self
+                // the row is in middle
+                let n_ties_of_last = self
                     .middle
-                    .range((middle_last_order_by.clone(), vec![])..)
+                    .range((middle_last_sort_key.clone(), vec![])..)
                     .count();
                 // We evict the last row and its ties only if the number of remaining rows is
                 // still larger than limit, i.e., there are limit-1 other rows.
@@ -724,38 +827,34 @@ impl AppendOnlyTopNCacheTrait for TopNCache<true> {
                 // insert 0 -> [0,1,1,1,1]
                 // insert 0 -> [0,0,1,1,1,1]
                 // insert 0 -> [0,0,0]
-                if self.middle.len() - num_ties + 1 >= self.limit {
+                if self.middle.len() + 1 - n_ties_of_last >= self.limit {
+                    // middle will be full without the last element and its ties after insertion
                     while let Some(middle_last) = self.middle.last_entry()
-                        && &middle_last.key().0 == middle_last_order_by
+                        && middle_last.key().0 == middle_last_sort_key
                     {
                         let middle_last = middle_last.remove_entry();
-                        res_ops.push(Op::Delete);
-                        res_rows.push(middle_last.1.clone());
+                        append_res(Op::Delete, middle_last.1.clone());
+
+                        // we don't need to maintain the high part so just delete it from state table
                         managed_state
                             .delete(row_deserializer.deserialize(middle_last.1.row.as_ref())?);
                     }
                 }
 
-                managed_state.insert(elem_to_compare_with_middle.1);
-                res_ops.push(Op::Insert);
-                res_rows.push((&elem_to_compare_with_middle.1).into());
-                self.middle.insert(
-                    elem_to_compare_with_middle.0,
-                    (&elem_to_compare_with_middle.1).into(),
-                );
+                managed_state.insert(to_insert.1);
+                let row: CompactedRow = to_insert.1.into();
+                append_res(Op::Insert, row.clone());
+                self.middle.insert(to_insert.0, row);
             }
             Ordering::Equal => {
-                // The row is in middle and is a tie with the last row.
-                managed_state.insert(elem_to_compare_with_middle.1);
-                res_ops.push(Op::Insert);
-                res_rows.push((&elem_to_compare_with_middle.1).into());
-                self.middle.insert(
-                    elem_to_compare_with_middle.0,
-                    (&elem_to_compare_with_middle.1).into(),
-                );
+                // the row is in middle and is a tie of the last row
+                managed_state.insert(to_insert.1);
+                let row: CompactedRow = to_insert.1.into();
+                append_res(Op::Insert, row.clone());
+                self.middle.insert(to_insert.0, row);
             }
             Ordering::Greater => {
-                // The row is in high. Do nothing.
+                // the row is in high, do nothing
             }
         }
 
diff --git a/src/stream/src/executor/top_n/top_n_plain.rs b/src/stream/src/executor/top_n/top_n_plain.rs
index 63b9ca94961f8..5c9370a9f35db 100644
--- a/src/stream/src/executor/top_n/top_n_plain.rs
+++ b/src/stream/src/executor/top_n/top_n_plain.rs
@@ -68,7 +68,7 @@ impl<S: StateStore> TopNExecutor<S, true> {
         let mut inner =
             InnerTopNExecutor::new(schema, storage_key, offset_and_limit, order_by, state_table)?;
 
-        inner.cache.high_capacity = 2;
+        inner.cache.high_cache_capacity = 2;
 
         Ok(TopNExecutorWrapper { input, ctx, inner })
     }
diff --git a/src/stream/src/executor/top_n/top_n_state.rs b/src/stream/src/executor/top_n/top_n_state.rs
index 57f4017e51185..52e5fd176a97f 100644
--- a/src/stream/src/executor/top_n/top_n_state.rs
+++ b/src/stream/src/executor/top_n/top_n_state.rs
@@ -16,16 +16,18 @@ use std::ops::Bound;
 use std::sync::Arc;
 
 use futures::{pin_mut, StreamExt};
-use risingwave_common::buffer::Bitmap;
+use risingwave_common::bitmap::Bitmap;
 use risingwave_common::row::{OwnedRow, Row, RowExt};
 use risingwave_common::types::ScalarImpl;
 use risingwave_common::util::epoch::EpochPair;
 use risingwave_storage::store::PrefetchOptions;
 use risingwave_storage::StateStore;
 
-use super::{serialize_pk_to_cache_key, CacheKey, CacheKeySerde, GroupKey, TopNCache};
+use super::top_n_cache::CacheKey;
+use super::{serialize_pk_to_cache_key, CacheKeySerde, GroupKey, TopNCache};
 use crate::common::table::state_table::StateTable;
 use crate::executor::error::StreamExecutorResult;
+use crate::executor::top_n::top_n_cache::Cache;
 
 /// * For TopN, the storage key is: `[ order_by + remaining columns of pk ]`
 /// * For group TopN, the storage key is: `[ group_key + order_by + remaining columns of pk ]`
@@ -76,8 +78,8 @@ impl<S: StateStore> ManagedTopNState<S> {
     }
 
     /// Update watermark for the managed state table.
-    pub fn update_watermark(&mut self, watermark: ScalarImpl, eager_cleaning: bool) {
-        self.state_table.update_watermark(watermark, eager_cleaning)
+    pub fn update_watermark(&mut self, watermark: ScalarImpl) {
+        self.state_table.update_watermark(watermark)
     }
 
     pub fn insert(&mut self, value: impl Row) {
@@ -143,8 +145,10 @@ impl<S: StateStore> ManagedTopNState<S> {
         start_key: Option<CacheKey>,
         cache_size_limit: usize,
     ) -> StreamExecutorResult<()> {
-        let cache = &mut topn_cache.high;
+        let high_cache = &mut topn_cache.high;
+        assert!(high_cache.is_empty());
 
+        // TODO(rc): iterate from `start_key`
         let sub_range: &(Bound<OwnedRow>, Bound<OwnedRow>) = &(Bound::Unbounded, Bound::Unbounded);
         let state_table_iter = self
             .state_table
@@ -171,13 +175,13 @@ impl<S: StateStore> ManagedTopNState<S> {
             {
                 continue;
             }
-            cache.insert(topn_row.cache_key, (&topn_row.row).into());
-            if cache.len() == cache_size_limit {
+            high_cache.insert(topn_row.cache_key, (&topn_row.row).into());
+            if high_cache.len() == cache_size_limit {
                 break;
             }
         }
 
-        if WITH_TIES && topn_cache.is_high_cache_full() {
+        if WITH_TIES && topn_cache.high_is_full() {
             let high_last_sort_key = topn_cache.high.last_key_value().unwrap().0 .0.clone();
             while let Some(item) = state_table_iter.next().await {
                 group_row_count += 1;
@@ -206,9 +210,10 @@ impl<S: StateStore> ManagedTopNState<S> {
         group_key: Option<impl GroupKey>,
         topn_cache: &mut TopNCache<WITH_TIES>,
     ) -> StreamExecutorResult<()> {
-        assert!(topn_cache.low.is_empty());
+        assert!(topn_cache.low.as_ref().map(Cache::is_empty).unwrap_or(true));
         assert!(topn_cache.middle.is_empty());
         assert!(topn_cache.high.is_empty());
+
         let sub_range: &(Bound<OwnedRow>, Bound<OwnedRow>) = &(Bound::Unbounded, Bound::Unbounded);
         let state_table_iter = self
             .state_table
@@ -225,14 +230,12 @@ impl<S: StateStore> ManagedTopNState<S> {
 
         let mut group_row_count = 0;
 
-        if topn_cache.offset > 0 {
+        if let Some(low) = &mut topn_cache.low {
             while let Some(item) = state_table_iter.next().await {
                 group_row_count += 1;
                 let topn_row = self.get_topn_row(item?.into_owned_row(), group_key.len());
-                topn_cache
-                    .low
-                    .insert(topn_row.cache_key, (&topn_row.row).into());
-                if topn_cache.low.len() == topn_cache.offset {
+                low.insert(topn_row.cache_key, (&topn_row.row).into());
+                if low.len() == topn_cache.offset {
                     break;
                 }
             }
@@ -249,7 +252,7 @@ impl<S: StateStore> ManagedTopNState<S> {
                 break;
             }
         }
-        if WITH_TIES && topn_cache.is_middle_cache_full() {
+        if WITH_TIES && topn_cache.middle_is_full() {
             let middle_last_sort_key = topn_cache.middle.last_key_value().unwrap().0 .0.clone();
             while let Some(item) = state_table_iter.next().await {
                 group_row_count += 1;
@@ -268,10 +271,10 @@ impl<S: StateStore> ManagedTopNState<S> {
         }
 
         assert!(
-            topn_cache.high_capacity > 0,
+            topn_cache.high_cache_capacity > 0,
             "topn cache high_capacity should always > 0"
         );
-        while !topn_cache.is_high_cache_full()
+        while !topn_cache.high_is_full()
             && let Some(item) = state_table_iter.next().await
         {
             group_row_count += 1;
@@ -280,7 +283,7 @@ impl<S: StateStore> ManagedTopNState<S> {
                 .high
                 .insert(topn_row.cache_key, (&topn_row.row).into());
         }
-        if WITH_TIES && topn_cache.is_high_cache_full() {
+        if WITH_TIES && topn_cache.high_is_full() {
             let high_last_sort_key = topn_cache.high.last_key_value().unwrap().0 .0.clone();
             while let Some(item) = state_table_iter.next().await {
                 group_row_count += 1;
diff --git a/src/stream/src/executor/top_n/topn_cache_state.rs b/src/stream/src/executor/top_n/topn_cache_state.rs
deleted file mode 100644
index fa28369f2b0d6..0000000000000
--- a/src/stream/src/executor/top_n/topn_cache_state.rs
+++ /dev/null
@@ -1,187 +0,0 @@
-// Copyright 2024 RisingWave Labs
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use core::fmt;
-use std::alloc::Global;
-use std::collections::btree_map::{BTreeMap, ExtractIf, OccupiedEntry, Range};
-use std::ops::RangeBounds;
-
-use risingwave_common::row::CompactedRow;
-use risingwave_common_estimate_size::{EstimateSize, KvSize};
-
-/// `CacheKey` is composed of `(order_by, remaining columns of pk)`.
-pub type CacheKey = (Vec<u8>, Vec<u8>);
-
-#[derive(Default)]
-pub struct TopNCacheState {
-    /// The full copy of the state.
-    inner: BTreeMap<CacheKey, CompactedRow>,
-    kv_heap_size: KvSize,
-}
-
-impl EstimateSize for TopNCacheState {
-    fn estimated_heap_size(&self) -> usize {
-        // TODO: Add btreemap internal size.
-        // https://github.com/risingwavelabs/risingwave/issues/9713
-        self.kv_heap_size.size()
-    }
-}
-
-impl TopNCacheState {
-    pub fn new() -> Self {
-        Self {
-            inner: BTreeMap::new(),
-            kv_heap_size: KvSize::new(),
-        }
-    }
-
-    /// Insert into the cache.
-    pub fn insert(&mut self, key: CacheKey, value: CompactedRow) -> Option<CompactedRow> {
-        self.kv_heap_size.add(&key, &value);
-        self.inner.insert(key, value)
-    }
-
-    /// Delete from the cache.
-    pub fn remove(&mut self, key: &CacheKey) {
-        if let Some(value) = self.inner.remove(key) {
-            self.kv_heap_size.sub(key, &value);
-        }
-    }
-
-    pub fn len(&self) -> usize {
-        self.inner.len()
-    }
-
-    pub fn is_empty(&self) -> bool {
-        self.inner.is_empty()
-    }
-
-    pub fn last_key_value(&self) -> Option<(&CacheKey, &CompactedRow)> {
-        self.inner.last_key_value()
-    }
-
-    pub fn first_key_value(&self) -> Option<(&CacheKey, &CompactedRow)> {
-        self.inner.first_key_value()
-    }
-
-    pub fn clear(&mut self) {
-        self.inner.clear()
-    }
-
-    pub fn pop_first(&mut self) -> Option<(CacheKey, CompactedRow)> {
-        self.inner.pop_first().inspect(|(k, v)| {
-            self.kv_heap_size.sub(k, v);
-        })
-    }
-
-    pub fn pop_last(&mut self) -> Option<(CacheKey, CompactedRow)> {
-        self.inner.pop_last().inspect(|(k, v)| {
-            self.kv_heap_size.sub(k, v);
-        })
-    }
-
-    pub fn last_entry(&mut self) -> Option<TopNCacheOccupiedEntry<'_>> {
-        self.inner
-            .last_entry()
-            .map(|entry| TopNCacheOccupiedEntry::new(entry, &mut self.kv_heap_size))
-    }
-
-    pub fn iter(&self) -> impl Iterator<Item = (&CacheKey, &CompactedRow)> {
-        self.inner.iter()
-    }
-
-    pub fn range<R>(&self, range: R) -> Range<'_, CacheKey, CompactedRow>
-    where
-        R: RangeBounds<CacheKey>,
-    {
-        self.inner.range(range)
-    }
-
-    pub fn extract_if<'a, F1>(
-        &'a mut self,
-        mut pred: F1,
-    ) -> TopNExtractIf<'a, impl FnMut(&CacheKey, &mut CompactedRow) -> bool>
-    where
-        F1: 'a + FnMut(&CacheKey, &CompactedRow) -> bool,
-    {
-        let pred_immut = move |key: &CacheKey, value: &mut CompactedRow| pred(key, value);
-        TopNExtractIf {
-            inner: self.inner.extract_if(pred_immut),
-            kv_heap_size: &mut self.kv_heap_size,
-        }
-    }
-
-    pub fn retain<F>(&mut self, mut f: F)
-    where
-        F: FnMut(&CacheKey, &CompactedRow) -> bool,
-    {
-        self.extract_if(|k, v| !f(k, v)).for_each(drop);
-    }
-}
-
-pub struct TopNCacheOccupiedEntry<'a> {
-    inner: OccupiedEntry<'a, CacheKey, CompactedRow>,
-    /// The total size of the `TopNCacheState`
-    kv_heap_size: &'a mut KvSize,
-}
-
-impl<'a> TopNCacheOccupiedEntry<'a> {
-    pub fn new(entry: OccupiedEntry<'a, CacheKey, CompactedRow>, size: &'a mut KvSize) -> Self {
-        Self {
-            inner: entry,
-            kv_heap_size: size,
-        }
-    }
-
-    pub fn remove_entry(self) -> (CacheKey, CompactedRow) {
-        let (k, v) = self.inner.remove_entry();
-        self.kv_heap_size.add(&k, &v);
-        (k, v)
-    }
-
-    pub fn key(&self) -> &CacheKey {
-        self.inner.key()
-    }
-}
-
-impl fmt::Debug for TopNCacheState {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        self.inner.fmt(f)
-    }
-}
-
-pub struct TopNExtractIf<'a, F>
-where
-    F: FnMut(&CacheKey, &mut CompactedRow) -> bool,
-{
-    inner: ExtractIf<'a, CacheKey, CompactedRow, F, Global>,
-    kv_heap_size: &'a mut KvSize,
-}
-
-impl<'a, F> Iterator for TopNExtractIf<'a, F>
-where
-    F: 'a + FnMut(&CacheKey, &mut CompactedRow) -> bool,
-{
-    type Item = (CacheKey, CompactedRow);
-
-    fn next(&mut self) -> Option<Self::Item> {
-        self.inner
-            .next()
-            .inspect(|(k, v)| self.kv_heap_size.sub(k, v))
-    }
-
-    fn size_hint(&self) -> (usize, Option<usize>) {
-        self.inner.size_hint()
-    }
-}
diff --git a/src/stream/src/executor/top_n/utils.rs b/src/stream/src/executor/top_n/utils.rs
index 18409cf3d6db1..207c6a6d9366a 100644
--- a/src/stream/src/executor/top_n/utils.rs
+++ b/src/stream/src/executor/top_n/utils.rs
@@ -16,14 +16,14 @@ use std::future::Future;
 
 use itertools::Itertools;
 use risingwave_common::array::Op;
-use risingwave_common::buffer::Bitmap;
+use risingwave_common::bitmap::Bitmap;
 use risingwave_common::row::{CompactedRow, RowDeserializer};
 use risingwave_common::util::chunk_coalesce::DataChunkBuilder;
 use risingwave_common::util::epoch::EpochPair;
 use risingwave_common::util::row_serde::OrderedRowSerde;
 use risingwave_common::util::sort_util::ColumnOrder;
 
-use super::CacheKey;
+use super::top_n_cache::CacheKey;
 use crate::executor::prelude::*;
 
 pub trait TopNExecutorBase: Send + 'static {
diff --git a/src/stream/src/executor/union.rs b/src/stream/src/executor/union.rs
index c3279f7e8a344..46acefb61d44f 100644
--- a/src/stream/src/executor/union.rs
+++ b/src/stream/src/executor/union.rs
@@ -164,7 +164,7 @@ async fn merge(
             None => {
                 assert!(active.is_terminated());
                 let barrier = current_barrier.take().unwrap();
-                barrier_align.observe(start_time.elapsed().as_secs_f64());
+                barrier_align.inc_by(start_time.elapsed().as_nanos() as u64);
 
                 let upstreams = std::mem::take(&mut blocked);
                 active.extend(upstreams.into_iter().map(|upstream| upstream.into_future()));
diff --git a/src/stream/src/executor/values.rs b/src/stream/src/executor/values.rs
index dfa5579d66b49..83da0ff68a7d5 100644
--- a/src/stream/src/executor/values.rs
+++ b/src/stream/src/executor/values.rs
@@ -121,7 +121,7 @@ impl ValuesExecutor {
 
         while let Some(barrier) = barrier_receiver.recv().await {
             if emit {
-                progress.finish(barrier.epoch.curr, 0);
+                progress.finish(barrier.epoch, 0);
             }
             yield Message::Barrier(barrier);
         }
@@ -207,6 +207,7 @@ mod tests {
                 added_actors: maplit::hashset! {actor_id},
                 splits: Default::default(),
                 pause: false,
+                subscriptions_to_add: vec![],
             }));
         tx.send(first_message).unwrap();
 
diff --git a/src/stream/src/executor/watermark_filter.rs b/src/stream/src/executor/watermark_filter.rs
index 2aca1251dd058..8f8b166626d21 100644
--- a/src/stream/src/executor/watermark_filter.rs
+++ b/src/stream/src/executor/watermark_filter.rs
@@ -77,6 +77,7 @@ impl<S: StateStore> Execute for WatermarkFilterExecutor<S> {
         self.execute_inner().boxed()
     }
 }
+const UPDATE_GLOBAL_WATERMARK_FREQUENCY_WHEN_IDLE: usize = 5;
 
 impl<S: StateStore> WatermarkFilterExecutor<S> {
     #[try_stream(ok = Message, error = StreamExecutorError)]
@@ -99,13 +100,18 @@ impl<S: StateStore> WatermarkFilterExecutor<S> {
         let mut input = input.execute();
 
         let first_barrier = expect_first_barrier(&mut input).await?;
+        let prev_epoch = first_barrier.epoch.prev;
         table.init_epoch(first_barrier.epoch);
         // The first barrier message should be propagated.
         yield Message::Barrier(first_barrier);
 
         // Initiate and yield the first watermark.
-        let mut current_watermark =
-            Self::get_global_max_watermark(&table, &global_watermark_table).await?;
+        let mut current_watermark = Self::get_global_max_watermark(
+            &table,
+            &global_watermark_table,
+            HummockReadEpoch::Committed(prev_epoch),
+        )
+        .await?;
 
         let mut last_checkpoint_watermark = None;
 
@@ -119,7 +125,7 @@ impl<S: StateStore> WatermarkFilterExecutor<S> {
 
         // If the input is idle
         let mut idle_input = true;
-
+        let mut barrier_num_during_idle = 0;
         #[for_await]
         for msg in input {
             let msg = msg?;
@@ -208,6 +214,9 @@ impl<S: StateStore> WatermarkFilterExecutor<S> {
                     }
                 }
                 Message::Barrier(barrier) => {
+                    let prev_epoch = barrier.epoch.prev;
+                    let is_checkpoint = barrier.kind.is_checkpoint();
+                    let mut need_update_global_max_watermark = false;
                     // Update the vnode bitmap for state tables of all agg calls if asked.
                     if let Some(vnode_bitmap) = barrier.as_update_vnode_bitmap(ctx.id) {
                         let other_vnodes_bitmap = Arc::new(
@@ -220,15 +229,11 @@ impl<S: StateStore> WatermarkFilterExecutor<S> {
 
                         // Take the global max watermark when scaling happens.
                         if previous_vnode_bitmap != vnode_bitmap {
-                            current_watermark =
-                                Self::get_global_max_watermark(&table, &global_watermark_table)
-                                    .await?;
+                            need_update_global_max_watermark = true;
                         }
                     }
 
-                    if barrier.kind.is_checkpoint()
-                        && last_checkpoint_watermark != current_watermark
-                    {
+                    if is_checkpoint && last_checkpoint_watermark != current_watermark {
                         last_checkpoint_watermark.clone_from(&current_watermark);
                         // Persist the watermark when checkpoint arrives.
                         if let Some(watermark) = current_watermark.clone() {
@@ -242,39 +247,59 @@ impl<S: StateStore> WatermarkFilterExecutor<S> {
                     }
 
                     table.commit(barrier.epoch).await?;
+                    yield Message::Barrier(barrier);
+
+                    if need_update_global_max_watermark {
+                        current_watermark = Self::get_global_max_watermark(
+                            &table,
+                            &global_watermark_table,
+                            HummockReadEpoch::Committed(prev_epoch),
+                        )
+                        .await?;
+                    }
 
-                    if barrier.kind.is_checkpoint() {
+                    if is_checkpoint {
                         if idle_input {
-                            // Align watermark
-                            let global_max_watermark =
-                                Self::get_global_max_watermark(&table, &global_watermark_table)
-                                    .await?;
-
-                            current_watermark = if let Some(global_max_watermark) =
-                                global_max_watermark.clone()
-                                && let Some(watermark) = current_watermark.clone()
+                            barrier_num_during_idle += 1;
+
+                            if barrier_num_during_idle
+                                == UPDATE_GLOBAL_WATERMARK_FREQUENCY_WHEN_IDLE
                             {
-                                Some(cmp::max_by(
-                                    watermark,
-                                    global_max_watermark,
-                                    DefaultOrd::default_cmp,
-                                ))
-                            } else {
-                                current_watermark.or(global_max_watermark)
-                            };
-                            if let Some(watermark) = current_watermark.clone() {
-                                yield Message::Watermark(Watermark::new(
-                                    event_time_col_idx,
-                                    watermark_type.clone(),
-                                    watermark,
-                                ));
+                                barrier_num_during_idle = 0;
+                                // Align watermark
+                                // NOTE(st1page): Should be `NoWait` because it could lead to a degradation of concurrent checkpoint situations, as it would require waiting for the previous epoch
+                                let global_max_watermark = Self::get_global_max_watermark(
+                                    &table,
+                                    &global_watermark_table,
+                                    HummockReadEpoch::NoWait(prev_epoch),
+                                )
+                                .await?;
+
+                                current_watermark = if let Some(global_max_watermark) =
+                                    global_max_watermark.clone()
+                                    && let Some(watermark) = current_watermark.clone()
+                                {
+                                    Some(cmp::max_by(
+                                        watermark,
+                                        global_max_watermark,
+                                        DefaultOrd::default_cmp,
+                                    ))
+                                } else {
+                                    current_watermark.or(global_max_watermark)
+                                };
+                                if let Some(watermark) = current_watermark.clone() {
+                                    yield Message::Watermark(Watermark::new(
+                                        event_time_col_idx,
+                                        watermark_type.clone(),
+                                        watermark,
+                                    ));
+                                }
                             }
                         } else {
                             idle_input = true;
+                            barrier_num_during_idle = 0;
                         }
                     }
-
-                    yield Message::Barrier(barrier);
                 }
             }
         }
@@ -301,14 +326,14 @@ impl<S: StateStore> WatermarkFilterExecutor<S> {
     async fn get_global_max_watermark(
         table: &StateTable<S>,
         global_watermark_table: &StorageTable<S>,
+        wait_epoch: HummockReadEpoch,
     ) -> StreamExecutorResult<Option<ScalarImpl>> {
-        let epoch = table.epoch();
         let handle_watermark_row = |watermark_row: Option<OwnedRow>| match watermark_row {
             Some(row) => {
                 if row.len() == 1 {
                     Ok::<_, StreamExecutorError>(row[0].to_owned())
                 } else {
-                    bail!("The watermark row should only contains 1 datum");
+                    bail!("The watermark row should only contain 1 datum");
                 }
             }
             _ => Ok(None),
@@ -319,9 +344,8 @@ impl<S: StateStore> WatermarkFilterExecutor<S> {
                 .iter_vnodes()
                 .map(|vnode| async move {
                     let pk = row::once(vnode.to_datum());
-                    let watermark_row: Option<OwnedRow> = global_watermark_table
-                        .get_row(pk, HummockReadEpoch::NoWait(epoch))
-                        .await?;
+                    let watermark_row: Option<OwnedRow> =
+                        global_watermark_table.get_row(pk, wait_epoch).await?;
                     handle_watermark_row(watermark_row)
                 });
         let local_watermark_iter_futures = table.vnodes().iter_vnodes().map(|vnode| async move {
diff --git a/src/stream/src/from_proto/agg_common.rs b/src/stream/src/from_proto/agg_common.rs
index 3799e50068e19..3bc928e1cdc7c 100644
--- a/src/stream/src/from_proto/agg_common.rs
+++ b/src/stream/src/from_proto/agg_common.rs
@@ -15,7 +15,7 @@
 use std::collections::HashMap;
 use std::sync::Arc;
 
-use risingwave_common::buffer::Bitmap;
+use risingwave_common::bitmap::Bitmap;
 use risingwave_common::util::sort_util::ColumnOrder;
 
 use super::*;
diff --git a/src/stream/src/from_proto/approx_percentile/global.rs b/src/stream/src/from_proto/approx_percentile/global.rs
new file mode 100644
index 0000000000000..f32c608472e60
--- /dev/null
+++ b/src/stream/src/from_proto/approx_percentile/global.rs
@@ -0,0 +1,55 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use risingwave_pb::stream_plan::GlobalApproxPercentileNode;
+
+use crate::common::table::state_table::StateTable;
+use crate::executor::GlobalApproxPercentileExecutor;
+use crate::from_proto::*;
+
+pub struct GlobalApproxPercentileExecutorBuilder;
+
+impl ExecutorBuilder for GlobalApproxPercentileExecutorBuilder {
+    type Node = GlobalApproxPercentileNode;
+
+    async fn new_boxed_executor(
+        params: ExecutorParams,
+        node: &Self::Node,
+        store: impl StateStore,
+    ) -> StreamResult<Executor> {
+        let [input]: [_; 1] = params.input.try_into().unwrap();
+        let bucket_table = node
+            .bucket_state_table
+            .as_ref()
+            .expect("bucket_state_table not provided");
+        let count_table = node
+            .count_state_table
+            .as_ref()
+            .expect("count_state_table not provided");
+        let bucket_state_table =
+            StateTable::from_table_catalog(bucket_table, store.clone(), None).await;
+        let count_state_table = StateTable::from_table_catalog(count_table, store, None).await;
+        let exec = GlobalApproxPercentileExecutor::new(
+            params.actor_context,
+            input,
+            node.quantile,
+            node.base,
+            params.env.config().developer.chunk_size,
+            bucket_state_table,
+            count_state_table,
+        )
+        .boxed();
+        Ok(Executor::new(params.info, exec))
+    }
+}
diff --git a/src/stream/src/from_proto/approx_percentile/local.rs b/src/stream/src/from_proto/approx_percentile/local.rs
new file mode 100644
index 0000000000000..df78799244132
--- /dev/null
+++ b/src/stream/src/from_proto/approx_percentile/local.rs
@@ -0,0 +1,41 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use risingwave_pb::stream_plan::LocalApproxPercentileNode;
+
+use crate::executor::LocalApproxPercentileExecutor;
+use crate::from_proto::*;
+
+pub struct LocalApproxPercentileExecutorBuilder;
+
+impl ExecutorBuilder for LocalApproxPercentileExecutorBuilder {
+    type Node = LocalApproxPercentileNode;
+
+    async fn new_boxed_executor(
+        params: ExecutorParams,
+        node: &Self::Node,
+        _store: impl StateStore,
+    ) -> StreamResult<Executor> {
+        let [input]: [_; 1] = params.input.try_into().unwrap();
+        let exec = LocalApproxPercentileExecutor::new(
+            params.actor_context,
+            input,
+            node.base,
+            node.percentile_index as usize,
+            params.env.config().developer.chunk_size,
+        )
+        .boxed();
+        Ok(Executor::new(params.info, exec))
+    }
+}
diff --git a/src/stream/src/from_proto/approx_percentile/mod.rs b/src/stream/src/from_proto/approx_percentile/mod.rs
new file mode 100644
index 0000000000000..29910d9032e15
--- /dev/null
+++ b/src/stream/src/from_proto/approx_percentile/mod.rs
@@ -0,0 +1,16 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub mod global;
+pub mod local;
diff --git a/src/stream/src/from_proto/dynamic_filter.rs b/src/stream/src/from_proto/dynamic_filter.rs
index c6ccd70382540..dbccf78e2aaf2 100644
--- a/src/stream/src/from_proto/dynamic_filter.rs
+++ b/src/stream/src/from_proto/dynamic_filter.rs
@@ -50,8 +50,6 @@ impl ExecutorBuilder for DynamicFilterExecutorBuilder {
             );
         }
 
-        let condition_always_relax = node.get_condition_always_relax();
-
         let state_table_r =
             StateTable::from_table_catalog(node.get_right_table()?, store.clone(), None).await;
 
@@ -75,7 +73,6 @@ impl ExecutorBuilder for DynamicFilterExecutorBuilder {
                 state_table_r,
                 params.executor_stats,
                 params.env.config().developer.chunk_size,
-                condition_always_relax,
                 cleaned_by_watermark,
             )
             .boxed()
@@ -95,7 +92,6 @@ impl ExecutorBuilder for DynamicFilterExecutorBuilder {
                 state_table_r,
                 params.executor_stats,
                 params.env.config().developer.chunk_size,
-                condition_always_relax,
                 cleaned_by_watermark,
             )
             .boxed()
diff --git a/src/stream/src/from_proto/mod.rs b/src/stream/src/from_proto/mod.rs
index caf1d72f4ef05..6f185695eadf7 100644
--- a/src/stream/src/from_proto/mod.rs
+++ b/src/stream/src/from_proto/mod.rs
@@ -53,6 +53,10 @@ mod union;
 mod values;
 mod watermark_filter;
 
+mod row_merge;
+
+mod approx_percentile;
+
 // import for submodules
 use itertools::Itertools;
 use risingwave_pb::stream_plan::stream_node::NodeBody;
@@ -60,6 +64,8 @@ use risingwave_pb::stream_plan::{StreamNode, TemporalJoinNode};
 use risingwave_storage::StateStore;
 
 use self::append_only_dedup::*;
+use self::approx_percentile::global::*;
+use self::approx_percentile::local::*;
 use self::barrier_recv::*;
 use self::batch_query::*;
 use self::cdc_filter::CdcFilterExecutorBuilder;
@@ -82,6 +88,7 @@ use self::over_window::*;
 use self::project::*;
 use self::project_set::*;
 use self::row_id_gen::RowIdGenExecutorBuilder;
+use self::row_merge::*;
 use self::simple_agg::*;
 use self::sink::*;
 use self::sort::*;
@@ -175,5 +182,8 @@ pub async fn create_executor(
         NodeBody::StreamFsFetch => FsFetchExecutorBuilder,
         NodeBody::SourceBackfill => SourceBackfillExecutorBuilder,
         NodeBody::Changelog => ChangeLogExecutorBuilder,
+        NodeBody::GlobalApproxPercentile => GlobalApproxPercentileExecutorBuilder,
+        NodeBody::LocalApproxPercentile => LocalApproxPercentileExecutorBuilder,
+        NodeBody::RowMerge => RowMergeExecutorBuilder,
     }
 }
diff --git a/src/stream/src/from_proto/row_merge.rs b/src/stream/src/from_proto/row_merge.rs
new file mode 100644
index 0000000000000..41e3620d4000f
--- /dev/null
+++ b/src/stream/src/from_proto/row_merge.rs
@@ -0,0 +1,45 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use risingwave_common::util::column_index_mapping::ColIndexMapping;
+use risingwave_pb::stream_plan::RowMergeNode;
+
+use crate::executor::RowMergeExecutor;
+use crate::from_proto::*;
+
+pub struct RowMergeExecutorBuilder;
+
+impl ExecutorBuilder for RowMergeExecutorBuilder {
+    type Node = RowMergeNode;
+
+    async fn new_boxed_executor(
+        params: ExecutorParams,
+        node: &Self::Node,
+        _store: impl StateStore,
+    ) -> StreamResult<Executor> {
+        let [lhs_input, rhs_input]: [_; 2] = params.input.try_into().unwrap();
+        let lhs_mapping = ColIndexMapping::from_protobuf(node.lhs_mapping.as_ref().unwrap());
+        let rhs_mapping = ColIndexMapping::from_protobuf(node.rhs_mapping.as_ref().unwrap());
+
+        let exec = RowMergeExecutor::new(
+            params.actor_context,
+            lhs_input,
+            rhs_input,
+            lhs_mapping,
+            rhs_mapping,
+            params.info.schema.clone(),
+        );
+        Ok(Executor::new(params.info, exec.boxed()))
+    }
+}
diff --git a/src/stream/src/from_proto/sink.rs b/src/stream/src/from_proto/sink.rs
index 5e77be7beb7a0..cff2009af7830 100644
--- a/src/stream/src/from_proto/sink.rs
+++ b/src/stream/src/from_proto/sink.rs
@@ -16,25 +16,54 @@ use std::sync::Arc;
 
 use anyhow::anyhow;
 use risingwave_common::catalog::{ColumnCatalog, Schema};
+use risingwave_common::secret::LocalSecretManager;
 use risingwave_common::types::DataType;
 use risingwave_connector::match_sink_name_str;
-use risingwave_connector::sink::catalog::{SinkFormatDesc, SinkType};
+use risingwave_connector::sink::catalog::{SinkFormatDesc, SinkId, SinkType};
 use risingwave_connector::sink::{
     SinkError, SinkMetaClient, SinkParam, SinkWriterParam, CONNECTOR_TYPE_KEY, SINK_TYPE_OPTION,
 };
 use risingwave_pb::catalog::Table;
 use risingwave_pb::plan_common::PbColumnCatalog;
 use risingwave_pb::stream_plan::{SinkLogStoreType, SinkNode};
+use risingwave_pb::telemetry::{PbTelemetryDatabaseObject, PbTelemetryEventStage};
 
 use super::*;
 use crate::common::log_store_impl::in_mem::BoundedInMemLogStoreFactory;
 use crate::common::log_store_impl::kv_log_store::{
     KvLogStoreFactory, KvLogStoreMetrics, KvLogStorePkInfo, KV_LOG_STORE_V2_INFO,
 };
-use crate::executor::SinkExecutor;
+use crate::executor::{SinkExecutor, StreamExecutorError};
+use crate::telemetry::report_event;
 
 pub struct SinkExecutorBuilder;
 
+fn telemetry_sink_build(
+    sink_id: &SinkId,
+    connector_name: &str,
+    sink_format_desc: &Option<SinkFormatDesc>,
+) {
+    let attr = sink_format_desc.as_ref().map(|f| {
+        let mut builder = jsonbb::Builder::<Vec<u8>>::new();
+        builder.begin_object();
+        builder.add_string("format");
+        builder.add_value(jsonbb::ValueRef::String(f.format.to_string().as_str()));
+        builder.add_string("encode");
+        builder.add_value(jsonbb::ValueRef::String(f.encode.to_string().as_str()));
+        builder.end_object();
+        builder.finish()
+    });
+
+    report_event(
+        PbTelemetryEventStage::CreateStreamJob,
+        "sink",
+        sink_id.sink_id() as i64,
+        Some(connector_name.to_string()),
+        Some(PbTelemetryDatabaseObject::Sink),
+        attr,
+    )
+}
+
 fn resolve_pk_info(
     input_schema: &Schema,
     log_store_table: &Table,
@@ -110,11 +139,12 @@ impl ExecutorBuilder for SinkExecutorBuilder {
 
         let sink_desc = node.sink_desc.as_ref().unwrap();
         let sink_type = SinkType::from_proto(sink_desc.get_sink_type().unwrap());
-        let sink_id = sink_desc.get_id().into();
+        let sink_id: SinkId = sink_desc.get_id().into();
         let sink_name = sink_desc.get_name().to_owned();
         let db_name = sink_desc.get_db_name().into();
         let sink_from_name = sink_desc.get_sink_from_name().into();
         let properties = sink_desc.get_properties().clone();
+        let secret_refs = sink_desc.get_secret_refs().clone();
         let downstream_pk = sink_desc
             .downstream_pk
             .iter()
@@ -129,7 +159,10 @@ impl ExecutorBuilder for SinkExecutorBuilder {
 
         let connector = {
             let sink_type = properties.get(CONNECTOR_TYPE_KEY).ok_or_else(|| {
-                SinkError::Config(anyhow!("missing config: {}", CONNECTOR_TYPE_KEY))
+                StreamExecutorError::from((
+                    SinkError::Config(anyhow!("missing config: {}", CONNECTOR_TYPE_KEY)),
+                    sink_id.sink_id,
+                ))
             })?;
 
             match_sink_name_str!(
@@ -137,28 +170,49 @@ impl ExecutorBuilder for SinkExecutorBuilder {
                 SinkType,
                 Ok(SinkType::SINK_NAME),
                 |other| {
-                    Err(SinkError::Config(anyhow!(
-                        "unsupported sink connector {}",
-                        other
+                    Err(StreamExecutorError::from((
+                        SinkError::Config(anyhow!("unsupported sink connector {}", other)),
+                        sink_id.sink_id,
                     )))
                 }
             )
         }?;
         let format_desc = match &sink_desc.format_desc {
             // Case A: new syntax `format ... encode ...`
-            Some(f) => Some(f.clone().try_into()?),
+            Some(f) => Some(
+                f.clone()
+                    .try_into()
+                    .map_err(|e| StreamExecutorError::from((e, sink_id.sink_id)))?,
+            ),
             None => match sink_desc.properties.get(SINK_TYPE_OPTION) {
                 // Case B: old syntax `type = '...'`
-                Some(t) => SinkFormatDesc::from_legacy_type(connector, t)?,
+                Some(t) => SinkFormatDesc::from_legacy_type(connector, t)
+                    .map_err(|e| StreamExecutorError::from((e, sink_id.sink_id)))?,
                 // Case C: no format + encode required
                 None => None,
             },
         };
 
+        let properties_with_secret =
+            LocalSecretManager::global().fill_secrets(properties, secret_refs)?;
+
+        let format_desc_with_secret = SinkParam::fill_secret_for_format_desc(format_desc)
+            .map_err(|e| StreamExecutorError::from((e, sink_id.sink_id)))?;
+
+        let actor_id_str = format!("{}", params.actor_context.id);
+        let sink_id_str = format!("{}", sink_id.sink_id);
+
+        let sink_metrics = params.executor_stats.new_sink_metrics(
+            &actor_id_str,
+            &sink_id_str,
+            &sink_name,
+            connector,
+        );
+
         let sink_param = SinkParam {
             sink_id,
             sink_name,
-            properties,
+            properties: properties_with_secret,
             columns: columns
                 .iter()
                 .filter(|col| !col.is_hidden)
@@ -166,19 +220,11 @@ impl ExecutorBuilder for SinkExecutorBuilder {
                 .collect(),
             downstream_pk,
             sink_type,
-            format_desc,
+            format_desc: format_desc_with_secret,
             db_name,
             sink_from_name,
         };
 
-        let sink_id_str = format!("{}", sink_id.sink_id);
-
-        let sink_metrics = params.executor_stats.new_sink_metrics(
-            &params.info.identity,
-            sink_id_str.as_str(),
-            connector,
-        );
-
         let sink_write_param = SinkWriterParam {
             executor_id: params.executor_id,
             vnode_bitmap: params.vnode_bitmap.clone(),
@@ -192,6 +238,8 @@ impl ExecutorBuilder for SinkExecutorBuilder {
             connector, sink_id.sink_id, params.executor_id
         );
 
+        telemetry_sink_build(&sink_id, connector, &sink_param.format_desc);
+
         let exec = match node.log_store_type() {
             // Default value is the normal in memory log store to be backward compatible with the
             // previously unset value
@@ -214,7 +262,7 @@ impl ExecutorBuilder for SinkExecutorBuilder {
             SinkLogStoreType::KvLogStore => {
                 let metrics = KvLogStoreMetrics::new(
                     &params.executor_stats,
-                    &params.info.identity,
+                    params.actor_context.id,
                     &sink_param,
                     connector,
                 );
diff --git a/src/stream/src/from_proto/source/fs_fetch.rs b/src/stream/src/from_proto/source/fs_fetch.rs
index 1951365a47eed..7a08c2d2f512e 100644
--- a/src/stream/src/from_proto/source/fs_fetch.rs
+++ b/src/stream/src/from_proto/source/fs_fetch.rs
@@ -20,6 +20,7 @@ use risingwave_connector::source::filesystem::opendal_source::{
 };
 use risingwave_connector::source::reader::desc::SourceDescBuilder;
 use risingwave_connector::source::ConnectorProperties;
+use risingwave_connector::WithOptionsSecResolved;
 use risingwave_pb::stream_plan::StreamFsFetchNode;
 use risingwave_storage::StateStore;
 
@@ -46,12 +47,14 @@ impl ExecutorBuilder for FsFetchExecutorBuilder {
         let source_id = TableId::new(source.source_id);
         let source_name = source.source_name.clone();
         let source_info = source.get_info()?;
-        let properties = ConnectorProperties::extract(source.with_properties.clone(), false)?;
+        let source_options_with_secret =
+            WithOptionsSecResolved::new(source.with_properties.clone(), source.secret_refs.clone());
+        let properties = ConnectorProperties::extract(source_options_with_secret.clone(), false)?;
         let source_desc_builder = SourceDescBuilder::new(
             source.columns.clone(),
             params.env.source_metrics(),
             source.row_id_index.map(|x| x as _),
-            source.with_properties.clone(),
+            source_options_with_secret,
             source_info.clone(),
             params.env.config().developer.connector_message_buffer_size,
             params.info.pk_indices.clone(),
diff --git a/src/stream/src/from_proto/source/mod.rs b/src/stream/src/from_proto/source/mod.rs
index 5383c8a768fd0..63a81ee961eb0 100644
--- a/src/stream/src/from_proto/source/mod.rs
+++ b/src/stream/src/from_proto/source/mod.rs
@@ -13,8 +13,50 @@
 // limitations under the License.
 
 mod trad_source;
+
+use std::collections::BTreeMap;
+
 pub use trad_source::{create_source_desc_builder, SourceExecutorBuilder};
 mod fs_fetch;
 pub use fs_fetch::FsFetchExecutorBuilder;
+use risingwave_common::catalog::TableId;
+use risingwave_connector::source::UPSTREAM_SOURCE_KEY;
+use risingwave_pb::catalog::PbStreamSourceInfo;
+use risingwave_pb::telemetry::{PbTelemetryDatabaseObject, PbTelemetryEventStage};
 
 use super::*;
+use crate::telemetry::report_event;
+
+fn get_connector_name(with_props: &BTreeMap<String, String>) -> String {
+    with_props
+        .get(UPSTREAM_SOURCE_KEY)
+        .map(|s| s.to_lowercase())
+        .unwrap_or_default()
+}
+
+fn telemetry_source_build(
+    source_type: &str, // "source" or "source backfill"
+    source_id: &TableId,
+    source_info: &PbStreamSourceInfo,
+    with_props: &BTreeMap<String, String>,
+) {
+    let mut builder = jsonbb::Builder::<Vec<u8>>::new();
+    builder.begin_object();
+    builder.add_string("format");
+    builder.add_value(jsonbb::ValueRef::String(source_info.format().as_str_name()));
+    builder.add_string("encode");
+    builder.add_value(jsonbb::ValueRef::String(
+        source_info.row_encode().as_str_name(),
+    ));
+    builder.end_object();
+    let value = builder.finish();
+
+    report_event(
+        PbTelemetryEventStage::CreateStreamJob,
+        source_type,
+        source_id.table_id as i64,
+        Some(get_connector_name(with_props)),
+        Some(PbTelemetryDatabaseObject::Source),
+        Some(value),
+    )
+}
diff --git a/src/stream/src/from_proto/source/trad_source.rs b/src/stream/src/from_proto/source/trad_source.rs
index 667f2c5d49bc9..39e3293933e4b 100644
--- a/src/stream/src/from_proto/source/trad_source.rs
+++ b/src/stream/src/from_proto/source/trad_source.rs
@@ -12,15 +12,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::BTreeMap;
-
 use risingwave_common::catalog::{
-    default_key_column_name_version_mapping, TableId, KAFKA_TIMESTAMP_COLUMN_NAME,
+    default_key_column_name_version_mapping, KAFKA_TIMESTAMP_COLUMN_NAME,
 };
 use risingwave_connector::source::reader::desc::SourceDescBuilder;
-use risingwave_connector::source::{should_copy_to_format_encode_options, UPSTREAM_SOURCE_KEY};
-use risingwave_connector::WithPropertiesExt;
-use risingwave_pb::catalog::PbStreamSourceInfo;
+use risingwave_connector::source::should_copy_to_format_encode_options;
+use risingwave_connector::{WithOptionsSecResolved, WithPropertiesExt};
 use risingwave_pb::data::data_type::TypeName as PbTypeName;
 use risingwave_pb::plan_common::additional_column::ColumnType as AdditionalColumnType;
 use risingwave_pb::plan_common::{
@@ -42,11 +39,13 @@ const FS_CONNECTORS: &[&str] = &["s3"];
 pub struct SourceExecutorBuilder;
 
 pub fn create_source_desc_builder(
+    source_type: &str, // "source" or "source backfill"
+    source_id: &TableId,
     mut source_columns: Vec<PbColumnCatalog>,
     params: &ExecutorParams,
     source_info: PbStreamSourceInfo,
     row_id_index: Option<u32>,
-    with_properties: BTreeMap<String, String>,
+    with_properties: WithOptionsSecResolved,
 ) -> SourceDescBuilder {
     {
         // compatible code: introduced in https://github.com/risingwavelabs/risingwave/pull/13707
@@ -112,6 +111,8 @@ pub fn create_source_desc_builder(
         });
     }
 
+    telemetry_source_build(source_type, source_id, &source_info, &with_properties);
+
     SourceDescBuilder::new(
         source_columns.clone(),
         params.env.source_metrics(),
@@ -155,11 +156,7 @@ impl ExecutorBuilder for SourceExecutorBuilder {
                 if source_info.format_encode_options.is_empty() {
                     // compatible code: quick fix for <https://github.com/risingwavelabs/risingwave/issues/14755>,
                     // will move the logic to FragmentManager::init in release 1.7.
-                    let connector = source
-                        .with_properties
-                        .get(UPSTREAM_SOURCE_KEY)
-                        .unwrap_or(&String::default())
-                        .to_owned();
+                    let connector = get_connector_name(&source.with_properties);
                     source_info.format_encode_options.extend(
                         source.with_properties.iter().filter_map(|(k, v)| {
                             should_copy_to_format_encode_options(k, &connector)
@@ -168,12 +165,19 @@ impl ExecutorBuilder for SourceExecutorBuilder {
                     );
                 }
 
+                let with_properties = WithOptionsSecResolved::new(
+                    source.with_properties.clone(),
+                    source.secret_refs.clone(),
+                );
+
                 let source_desc_builder = create_source_desc_builder(
+                    "source",
+                    &source_id,
                     source.columns.clone(),
                     &params,
                     source_info,
                     source.row_id_index,
-                    source.with_properties.clone(),
+                    with_properties,
                 );
 
                 let source_column_ids: Vec<_> = source_desc_builder
@@ -195,11 +199,7 @@ impl ExecutorBuilder for SourceExecutorBuilder {
                     state_table_handler,
                 );
 
-                let connector = source
-                    .with_properties
-                    .get("connector")
-                    .map(|c| c.to_ascii_lowercase())
-                    .unwrap_or_default();
+                let connector = get_connector_name(&source.with_properties);
                 let is_fs_connector = FS_CONNECTORS.contains(&connector.as_str());
                 let is_fs_v2_connector = source.with_properties.is_new_fs_connector();
 
diff --git a/src/stream/src/from_proto/source_backfill.rs b/src/stream/src/from_proto/source_backfill.rs
index 84f4bf7adab86..ba3ab599af700 100644
--- a/src/stream/src/from_proto/source_backfill.rs
+++ b/src/stream/src/from_proto/source_backfill.rs
@@ -13,6 +13,7 @@
 // limitations under the License.
 
 use risingwave_common::catalog::TableId;
+use risingwave_connector::WithOptionsSecResolved;
 use risingwave_pb::stream_plan::SourceBackfillNode;
 
 use super::*;
@@ -35,12 +36,16 @@ impl ExecutorBuilder for SourceBackfillExecutorBuilder {
         let source_name = node.source_name.clone();
         let source_info = node.get_info()?;
 
+        let options_with_secret =
+            WithOptionsSecResolved::new(node.with_properties.clone(), node.secret_refs.clone());
         let source_desc_builder = super::source::create_source_desc_builder(
+            "source backfill",
+            &source_id,
             node.columns.clone(),
             &params,
             source_info.clone(),
             node.row_id_index,
-            node.with_properties.clone(),
+            options_with_secret,
         );
 
         let source_column_ids: Vec<_> = source_desc_builder
diff --git a/src/stream/src/from_proto/stream_cdc_scan.rs b/src/stream/src/from_proto/stream_cdc_scan.rs
index 150812c57a1c4..3c81ecb80e859 100644
--- a/src/stream/src/from_proto/stream_cdc_scan.rs
+++ b/src/stream/src/from_proto/stream_cdc_scan.rs
@@ -12,10 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::HashMap;
 use std::sync::Arc;
 
-use anyhow::anyhow;
+use anyhow::Context;
 use risingwave_common::catalog::{Schema, TableId};
 use risingwave_common::util::sort_util::OrderType;
 use risingwave_connector::source::cdc::external::{
@@ -52,11 +51,7 @@ impl ExecutorBuilder for StreamCdcScanExecutorBuilder {
         assert_eq!(output_indices, (0..output_schema.len()).collect_vec());
         assert_eq!(output_schema.data_types(), params.info.schema.data_types());
 
-        let properties: HashMap<String, String> = table_desc
-            .connect_properties
-            .iter()
-            .map(|(k, v)| (k.clone(), v.clone()))
-            .collect();
+        let properties = table_desc.connect_properties.clone();
 
         let table_pk_order_types = table_desc
             .pk
@@ -92,18 +87,15 @@ impl ExecutorBuilder for StreamCdcScanExecutorBuilder {
             .collect();
 
         let schema_table_name = SchemaTableName::from_properties(&properties);
-        let table_config = serde_json::from_value::<ExternalTableConfig>(
-            serde_json::to_value(properties).unwrap(),
+        let table_config = ExternalTableConfig::try_from_btreemap(
+            properties.clone(),
+            table_desc.secret_refs.clone(),
         )
-        .map_err(|e| anyhow!("failed to parse external table config").context(e))?;
+        .context("failed to parse external table config")?;
+
         let database_name = table_config.database.clone();
         let table_reader = table_type
-            .create_table_reader(
-                table_config,
-                table_schema.clone(),
-                table_pk_indices.clone(),
-                scan_options.snapshot_batch_size,
-            )
+            .create_table_reader(table_config, table_schema.clone(), table_pk_indices.clone())
             .await?;
 
         let external_table = ExternalStorageTable::new(
diff --git a/src/stream/src/lib.rs b/src/stream/src/lib.rs
index a199e4c8acb9f..876deabc80f98 100644
--- a/src/stream/src/lib.rs
+++ b/src/stream/src/lib.rs
@@ -29,7 +29,6 @@
 #![feature(map_try_insert)]
 #![feature(never_type)]
 #![feature(btreemap_alloc)]
-#![feature(lazy_cell)]
 #![feature(error_generic_member_access)]
 #![feature(btree_extract_if)]
 #![feature(iter_order_by)]
@@ -55,6 +54,7 @@ pub mod error;
 pub mod executor;
 mod from_proto;
 pub mod task;
+pub mod telemetry;
 
 #[cfg(test)]
 risingwave_expr_impl::enable!();
diff --git a/src/stream/src/task/barrier_manager.rs b/src/stream/src/task/barrier_manager.rs
index 1098f505004c4..654980db1c17e 100644
--- a/src/stream/src/task/barrier_manager.rs
+++ b/src/stream/src/task/barrier_manager.rs
@@ -23,6 +23,7 @@ use futures::stream::{BoxStream, FuturesUnordered};
 use futures::StreamExt;
 use itertools::Itertools;
 use parking_lot::Mutex;
+use risingwave_common::error::tonic::extra::Score;
 use risingwave_pb::stream_service::barrier_complete_response::{
     GroupedSstableInfo, PbCreateMviewProgress,
 };
@@ -31,14 +32,15 @@ use rw_futures_util::{pending_on_none, AttachedFuture};
 use thiserror_ext::AsReport;
 use tokio::select;
 use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender};
-use tokio::sync::oneshot;
+use tokio::sync::{mpsc, oneshot};
 use tokio::task::JoinHandle;
 use tonic::{Code, Status};
 
 use self::managed_state::ManagedBarrierState;
 use crate::error::{IntoUnexpectedExit, StreamError, StreamResult};
 use crate::task::{
-    ActorHandle, ActorId, AtomicU64Ref, SharedContext, StreamEnvironment, UpDownActorIds,
+    ActorHandle, ActorId, AtomicU64Ref, PartialGraphId, SharedContext, StreamEnvironment,
+    UpDownActorIds,
 };
 
 mod managed_state;
@@ -48,13 +50,16 @@ mod tests;
 
 pub use progress::CreateMviewProgress;
 use risingwave_common::catalog::TableId;
+use risingwave_common::util::epoch::EpochPair;
 use risingwave_common::util::runtime::BackgroundShutdownRuntime;
 use risingwave_hummock_sdk::table_stats::to_prost_table_stats_map;
 use risingwave_hummock_sdk::{LocalSstableInfo, SyncResult};
 use risingwave_pb::common::ActorInfo;
 use risingwave_pb::stream_plan::barrier::BarrierKind;
 use risingwave_pb::stream_service::streaming_control_stream_request::{InitRequest, Request};
-use risingwave_pb::stream_service::streaming_control_stream_response::InitResponse;
+use risingwave_pb::stream_service::streaming_control_stream_response::{
+    InitResponse, ShutdownResponse,
+};
 use risingwave_pb::stream_service::{
     streaming_control_stream_response, BarrierCompleteResponse, BuildActorInfo,
     StreamingControlStreamRequest, StreamingControlStreamResponse,
@@ -62,7 +67,10 @@ use risingwave_pb::stream_service::{
 
 use crate::executor::exchange::permit::Receiver;
 use crate::executor::monitor::StreamingMetrics;
-use crate::executor::{Actor, Barrier, DispatchExecutor, Mutation, StreamExecutorError};
+use crate::executor::{
+    Actor, Barrier, BarrierInner, DispatchExecutor, DispatcherBarrier, Mutation,
+    StreamExecutorError,
+};
 use crate::task::barrier_manager::managed_state::ManagedBarrierStateDebugInfo;
 use crate::task::barrier_manager::progress::BackfillState;
 
@@ -114,14 +122,37 @@ impl ControlStreamHandle {
 
             let err = err.into_inner();
             if sender.send(Err(err)).is_err() {
-                warn!("failed to notify finish of control stream");
+                warn!("failed to notify reset of control stream");
             }
         }
     }
 
-    fn inspect_result(&mut self, result: StreamResult<()>) {
-        if let Err(e) = result {
-            self.reset_stream_with_err(e.to_status_unnamed(Code::Internal));
+    /// Send `Shutdown` message to the control stream and wait for the stream to be closed
+    /// by the meta service.
+    async fn shutdown_stream(&mut self) {
+        if let Some((sender, _)) = self.pair.take() {
+            if sender
+                .send(Ok(StreamingControlStreamResponse {
+                    response: Some(streaming_control_stream_response::Response::Shutdown(
+                        ShutdownResponse::default(),
+                    )),
+                }))
+                .is_err()
+            {
+                warn!("failed to notify shutdown of control stream");
+            } else {
+                tracing::info!("waiting for meta service to close control stream...");
+
+                // Wait for the stream to be closed, to ensure that the `Shutdown` message has
+                // been acknowledged by the meta service for more precise error report.
+                //
+                // This is because the meta service will reset the control stream manager and
+                // drop the connection to us upon recovery. As a result, the receiver part of
+                // this sender will also be dropped, causing the stream to close.
+                sender.closed().await;
+            }
+        } else {
+            debug!("control stream has been reset, ignore shutdown");
         }
     }
 
@@ -187,24 +218,28 @@ impl CreateActorContext {
     }
 }
 
+pub(crate) type SubscribeMutationItem = (u64, Option<Arc<Mutation>>);
+
 pub(super) enum LocalBarrierEvent {
     ReportActorCollected {
         actor_id: ActorId,
-        barrier: Barrier,
+        epoch: EpochPair,
     },
     ReportCreateProgress {
-        current_epoch: u64,
+        epoch: EpochPair,
         actor: ActorId,
         state: BackfillState,
     },
-    ReadBarrierMutation {
-        barrier: Barrier,
-        mutation_sender: oneshot::Sender<Option<Arc<Mutation>>>,
+    SubscribeBarrierMutation {
+        actor_id: ActorId,
+        epoch: EpochPair,
+        mutation_sender: mpsc::UnboundedSender<SubscribeMutationItem>,
     },
     #[cfg(test)]
     Flush(oneshot::Sender<()>),
 }
 
+#[derive(strum_macros::Display)]
 pub(super) enum LocalActorOperation {
     NewControlStream {
         handle: ControlStreamHandle,
@@ -241,6 +276,9 @@ pub(super) enum LocalActorOperation {
     InspectState {
         result_sender: oneshot::Sender<String>,
     },
+    Shutdown {
+        result_sender: oneshot::Sender<()>,
+    },
 }
 
 pub(super) struct CreateActorOutput {
@@ -370,7 +408,8 @@ pub(super) struct LocalBarrierWorker {
 
     actor_failure_rx: UnboundedReceiver<(ActorId, StreamError)>,
 
-    root_failure: Option<StreamError>,
+    /// Cached result of [`Self::try_find_root_failure`].
+    cached_root_failure: Option<ScoredStreamError>,
 }
 
 impl LocalBarrierWorker {
@@ -378,8 +417,7 @@ impl LocalBarrierWorker {
         let (event_tx, event_rx) = unbounded_channel();
         let (failure_tx, failure_rx) = unbounded_channel();
         let shared_context = Arc::new(SharedContext::new(
-            actor_manager.env.server_address().clone(),
-            actor_manager.env.config(),
+            &actor_manager.env,
             LocalBarrierManager {
                 barrier_event_sender: event_tx,
                 actor_failure_sender: failure_tx,
@@ -399,7 +437,7 @@ impl LocalBarrierWorker {
             current_shared_context: shared_context,
             barrier_event_rx: event_rx,
             actor_failure_rx: failure_rx,
-            root_failure: None,
+            cached_root_failure: None,
         }
     }
 
@@ -425,16 +463,18 @@ impl LocalBarrierWorker {
                 (sender, create_actors_result) = self.actor_manager_state.next_created_actors() => {
                     self.handle_actor_created(sender, create_actors_result);
                 }
-                completed_epoch = self.state.next_completed_epoch() => {
-                    let result = self.on_epoch_completed(completed_epoch);
-                    self.control_stream_handle.inspect_result(result);
+                (partial_graph_id, completed_epoch) = self.state.next_completed_epoch() => {
+                    let result = self.on_epoch_completed(partial_graph_id, completed_epoch);
+                    if let Err(err) = result {
+                        self.notify_other_failure(err, "failed to complete epoch").await;
+                    }
                 },
                 event = self.barrier_event_rx.recv() => {
                     self.handle_barrier_event(event.expect("should not be none"));
                 },
                 failure = self.actor_failure_rx.recv() => {
                     let (actor_id, err) = failure.unwrap();
-                    self.notify_failure(actor_id, err).await;
+                    self.notify_actor_failure(actor_id, err).await;
                 },
                 actor_op = actor_op_rx.recv() => {
                     if let Some(actor_op) = actor_op {
@@ -447,6 +487,15 @@ impl LocalBarrierWorker {
                                     response: Some(streaming_control_stream_response::Response::Init(InitResponse {}))
                                 });
                             }
+                            LocalActorOperation::Shutdown { result_sender } => {
+                                if !self.actor_manager_state.handles.is_empty() {
+                                    tracing::warn!(
+                                        "shutdown with running actors, scaling or migration will be triggered"
+                                    );
+                                }
+                                self.control_stream_handle.shutdown_stream().await;
+                                let _ = result_sender.send(());
+                            }
                             actor_op => {
                                 self.handle_actor_op(actor_op);
                             }
@@ -458,7 +507,9 @@ impl LocalBarrierWorker {
                 },
                 request = self.control_stream_handle.next_request() => {
                     let result = self.handle_streaming_control_request(request);
-                    self.control_stream_handle.inspect_result(result);
+                    if let Err(err) = result {
+                        self.notify_other_failure(err, "failed to inject barrier").await;
+                    }
                 },
             }
         }
@@ -494,9 +545,16 @@ impl LocalBarrierWorker {
                         .into_iter()
                         .map(TableId::new)
                         .collect(),
+                    PartialGraphId::new(req.partial_graph_id),
                 )?;
                 Ok(())
             }
+            Request::RemovePartialGraph(req) => {
+                self.remove_partial_graphs(
+                    req.partial_graph_ids.into_iter().map(PartialGraphId::new),
+                );
+                Ok(())
+            }
             Request::Init(_) => {
                 unreachable!()
             }
@@ -505,21 +563,23 @@ impl LocalBarrierWorker {
 
     fn handle_barrier_event(&mut self, event: LocalBarrierEvent) {
         match event {
-            LocalBarrierEvent::ReportActorCollected { actor_id, barrier } => {
-                self.collect(actor_id, &barrier)
+            LocalBarrierEvent::ReportActorCollected { actor_id, epoch } => {
+                self.collect(actor_id, epoch)
             }
             LocalBarrierEvent::ReportCreateProgress {
-                current_epoch,
+                epoch,
                 actor,
                 state,
             } => {
-                self.update_create_mview_progress(current_epoch, actor, state);
+                self.update_create_mview_progress(epoch, actor, state);
             }
-            LocalBarrierEvent::ReadBarrierMutation {
-                barrier,
+            LocalBarrierEvent::SubscribeBarrierMutation {
+                actor_id,
+                epoch,
                 mutation_sender,
             } => {
-                self.read_barrier_mutation(barrier, mutation_sender);
+                self.state
+                    .subscribe_actor_mutation(actor_id, epoch.prev, mutation_sender);
             }
             #[cfg(test)]
             LocalBarrierEvent::Flush(sender) => sender.send(()).unwrap(),
@@ -528,8 +588,8 @@ impl LocalBarrierWorker {
 
     fn handle_actor_op(&mut self, actor_op: LocalActorOperation) {
         match actor_op {
-            LocalActorOperation::NewControlStream { .. } => {
-                unreachable!("NewControlStream event should be handled separately in async context")
+            LocalActorOperation::NewControlStream { .. } | LocalActorOperation::Shutdown { .. } => {
+                unreachable!("event {actor_op} should be handled separately in async context")
             }
             LocalActorOperation::DropActors {
                 actors,
@@ -581,9 +641,17 @@ impl LocalBarrierWorker {
 
 // event handler
 impl LocalBarrierWorker {
-    fn on_epoch_completed(&mut self, epoch: u64) -> StreamResult<()> {
-        let result = self
+    fn on_epoch_completed(
+        &mut self,
+        partial_graph_id: PartialGraphId,
+        epoch: u64,
+    ) -> StreamResult<()> {
+        let state = self
             .state
+            .graph_states
+            .get_mut(&partial_graph_id)
+            .expect("should exist");
+        let result = state
             .pop_completed_epoch(epoch)
             .expect("should exist")
             .expect("should have completed")?;
@@ -608,18 +676,18 @@ impl LocalBarrierWorker {
                 streaming_control_stream_response::Response::CompleteBarrier(
                     BarrierCompleteResponse {
                         request_id: "todo".to_string(),
+                        partial_graph_id: partial_graph_id.into(),
+                        epoch,
                         status: None,
                         create_mview_progress,
                         synced_sstables: synced_sstables
                             .into_iter()
                             .map(
                                 |LocalSstableInfo {
-                                     compaction_group_id,
                                      sst_info,
                                      table_stats,
                                  }| GroupedSstableInfo {
-                                    compaction_group_id,
-                                    sst: Some(sst_info),
+                                    sst: Some(sst_info.into()),
                                     table_stats_map: to_prost_table_stats_map(table_stats),
                                 },
                             )
@@ -627,11 +695,11 @@ impl LocalBarrierWorker {
                         worker_id: self.actor_manager.env.worker_id(),
                         table_watermarks: table_watermarks
                             .into_iter()
-                            .map(|(key, value)| (key.table_id, value.to_protobuf()))
+                            .map(|(key, value)| (key.table_id, value.into()))
                             .collect(),
                         old_value_sstables: old_value_ssts
                             .into_iter()
-                            .map(|sst| sst.sst_info)
+                            .map(|sst| sst.sst_info.into())
                             .collect(),
                     },
                 ),
@@ -642,15 +710,6 @@ impl LocalBarrierWorker {
         Ok(())
     }
 
-    /// Read mutation from barrier state.
-    fn read_barrier_mutation(
-        &mut self,
-        barrier: Barrier,
-        sender: oneshot::Sender<Option<Arc<Mutation>>>,
-    ) {
-        self.state.read_barrier_mutation(&barrier, sender);
-    }
-
     /// Register sender for source actors, used to send barriers.
     fn register_sender(&mut self, actor_id: ActorId, senders: Vec<UnboundedSender<Barrier>>) {
         tracing::debug!(
@@ -666,15 +725,19 @@ impl LocalBarrierWorker {
 
     /// Broadcast a barrier to all senders. Save a receiver which will get notified when this
     /// barrier is finished, in managed mode.
+    ///
+    /// Note that the error returned here is typically a [`StreamError::barrier_send`], which is not
+    /// the root cause of the failure. The caller should then call [`Self::try_find_root_failure`]
+    /// to find the root cause.
     fn send_barrier(
         &mut self,
         barrier: &Barrier,
         to_send: HashSet<ActorId>,
         to_collect: HashSet<ActorId>,
         table_ids: HashSet<TableId>,
+        partial_graph_id: PartialGraphId,
     ) -> StreamResult<()> {
-        #[cfg(not(test))]
-        {
+        if !cfg!(test) {
             // The barrier might be outdated and been injected after recovery in some certain extreme
             // scenarios. So some newly creating actors in the barrier are possibly not rebuilt during
             // recovery. Check it here and return an error here if some actors are not found to
@@ -706,21 +769,21 @@ impl LocalBarrierWorker {
             to_collect
         );
 
-        // There must be some actors to collect from.
-        assert!(!to_collect.is_empty());
-
         for actor_id in &to_collect {
-            if let Some(e) = self.failure_actors.get(actor_id) {
+            if self.failure_actors.contains_key(actor_id) {
                 // The failure actors could exit before the barrier is issued, while their
                 // up-downstream actors could be stuck somehow. Return error directly to trigger the
                 // recovery.
-                // try_find_root_failure is not used merely because it requires async.
-                return Err(self.root_failure.clone().unwrap_or(e.clone()));
+                return Err(StreamError::barrier_send(
+                    barrier.clone(),
+                    *actor_id,
+                    "actor has already failed",
+                ));
             }
         }
 
         self.state
-            .transform_to_issued(barrier, to_collect, table_ids);
+            .transform_to_issued(barrier, to_collect, table_ids, partial_graph_id);
 
         for actor_id in to_send {
             match self.barrier_senders.get(&actor_id) {
@@ -760,6 +823,23 @@ impl LocalBarrierWorker {
         Ok(())
     }
 
+    fn remove_partial_graphs(&mut self, partial_graph_ids: impl Iterator<Item = PartialGraphId>) {
+        for partial_graph_id in partial_graph_ids {
+            if let Some(graph) = self.state.graph_states.remove(&partial_graph_id) {
+                assert!(
+                    graph.is_empty(),
+                    "non empty graph to be removed: {}",
+                    &graph
+                );
+            } else {
+                warn!(
+                    partial_graph_id = partial_graph_id.0,
+                    "no partial graph to remove"
+                );
+            }
+        }
+    }
+
     /// Reset all internal states.
     pub(super) fn reset_state(&mut self) {
         *self = Self::new(self.actor_manager.clone());
@@ -767,29 +847,45 @@ impl LocalBarrierWorker {
 
     /// When a [`crate::executor::StreamConsumer`] (typically [`crate::executor::DispatchExecutor`]) get a barrier, it should report
     /// and collect this barrier with its own `actor_id` using this function.
-    fn collect(&mut self, actor_id: ActorId, barrier: &Barrier) {
-        self.state.collect(actor_id, barrier)
+    fn collect(&mut self, actor_id: ActorId, epoch: EpochPair) {
+        self.state.collect(actor_id, epoch)
     }
 
-    /// When a actor exit unexpectedly, it should report this event using this function, so meta
-    /// will notice actor's exit while collecting.
-    async fn notify_failure(&mut self, actor_id: ActorId, err: StreamError) {
+    /// When a actor exit unexpectedly, the error is reported using this function. The control stream
+    /// will be reset and the meta service will then trigger recovery.
+    async fn notify_actor_failure(&mut self, actor_id: ActorId, err: StreamError) {
         self.add_failure(actor_id, err.clone());
-        let root_err = self.try_find_root_failure().await;
+        let root_err = self.try_find_root_failure().await.unwrap(); // always `Some` because we just added one
 
-        let failed_epochs = self.state.epochs_await_on_actor(actor_id).collect_vec();
-        if !failed_epochs.is_empty() {
+        if let Some(actor_state) = self.state.actor_states.get(&actor_id)
+            && (!actor_state.inflight_barriers.is_empty() || actor_state.is_running())
+        {
             self.control_stream_handle.reset_stream_with_err(
                 anyhow!(root_err)
-                    .context(format!(
-                        "failed to collect barrier for epoch {:?}",
-                        failed_epochs
-                    ))
+                    .context("failed to collect barrier")
                     .to_status_unnamed(Code::Internal),
             );
         }
     }
 
+    /// When some other failure happens (like failed to send barrier), the error is reported using
+    /// this function. The control stream will be reset and the meta service will then trigger recovery.
+    ///
+    /// This is similar to [`Self::notify_actor_failure`], but since there's not always an actor failure,
+    /// the given `err` will be used if there's no root failure found.
+    async fn notify_other_failure(&mut self, err: StreamError, message: impl Into<String>) {
+        let root_err = self
+            .try_find_root_failure()
+            .await
+            .unwrap_or_else(|| ScoredStreamError::new(err));
+
+        self.control_stream_handle.reset_stream_with_err(
+            anyhow!(root_err)
+                .context(message.into())
+                .to_status_unnamed(Code::Internal),
+        );
+    }
+
     fn add_failure(&mut self, actor_id: ActorId, err: StreamError) {
         if let Some(prev_err) = self.failure_actors.insert(actor_id, err) {
             warn!(
@@ -800,10 +896,14 @@ impl LocalBarrierWorker {
         }
     }
 
-    async fn try_find_root_failure(&mut self) -> StreamError {
-        if let Some(root_failure) = &self.root_failure {
-            return root_failure.clone();
+    /// Collect actor errors for a while and find the one that might be the root cause.
+    ///
+    /// Returns `None` if there's no actor error received.
+    async fn try_find_root_failure(&mut self) -> Option<ScoredStreamError> {
+        if self.cached_root_failure.is_some() {
+            return self.cached_root_failure.clone();
         }
+
         // fetch more actor errors within a timeout
         let _ = tokio::time::timeout(Duration::from_secs(3), async {
             while let Some((actor_id, error)) = self.actor_failure_rx.recv().await {
@@ -811,11 +911,15 @@ impl LocalBarrierWorker {
             }
         })
         .await;
-        self.root_failure = try_find_root_actor_failure(self.failure_actors.values());
 
-        self.root_failure
-            .clone()
-            .expect("failure actors should not be empty")
+        // Find the error with highest score.
+        self.cached_root_failure = self
+            .failure_actors
+            .values()
+            .map(|e| ScoredStreamError::new(e.clone()))
+            .max_by_key(|e| e.score);
+
+        self.cached_root_failure.clone()
     }
 }
 
@@ -891,10 +995,10 @@ impl LocalBarrierManager {
 
     /// When a [`crate::executor::StreamConsumer`] (typically [`crate::executor::DispatchExecutor`]) get a barrier, it should report
     /// and collect this barrier with its own `actor_id` using this function.
-    pub fn collect(&self, actor_id: ActorId, barrier: &Barrier) {
+    pub fn collect<M>(&self, actor_id: ActorId, barrier: &BarrierInner<M>) {
         self.send_event(LocalBarrierEvent::ReportActorCollected {
             actor_id,
-            barrier: barrier.clone(),
+            epoch: barrier.epoch,
         })
     }
 
@@ -907,79 +1011,103 @@ impl LocalBarrierManager {
     }
 
     /// When a `RemoteInput` get a barrier, it should wait and read the barrier mutation from the barrier manager.
-    pub async fn read_barrier_mutation(
+    pub fn subscribe_barrier_mutation(
         &self,
-        barrier: &Barrier,
-    ) -> StreamResult<Option<Arc<Mutation>>> {
-        let (tx, rx) = oneshot::channel();
-        self.send_event(LocalBarrierEvent::ReadBarrierMutation {
-            barrier: barrier.clone(),
+        actor_id: ActorId,
+        first_barrier: &DispatcherBarrier,
+    ) -> mpsc::UnboundedReceiver<SubscribeMutationItem> {
+        let (tx, rx) = mpsc::unbounded_channel();
+        self.send_event(LocalBarrierEvent::SubscribeBarrierMutation {
+            actor_id,
+            epoch: first_barrier.epoch,
             mutation_sender: tx,
         });
-        rx.await
-            .map_err(|_| anyhow!("barrier manager maybe reset").into())
+        rx
     }
 }
 
-/// Tries to find the root cause of actor failures, based on hard-coded rules.
-pub fn try_find_root_actor_failure<'a>(
-    actor_errors: impl IntoIterator<Item = &'a StreamError>,
-) -> Option<StreamError> {
-    // Explicitly list all error kinds here to notice developers to update this function when
-    // there are changes in error kinds.
-
-    fn stream_executor_error_score(e: &StreamExecutorError) -> i32 {
-        use crate::executor::error::ErrorKind;
-        match e.inner() {
-            // `ChannelClosed` or `ExchangeChannelClosed` is likely to be caused by actor exit
-            // and not the root cause.
-            ErrorKind::ChannelClosed(_) | ErrorKind::ExchangeChannelClosed(_) => 1,
-
-            // Normal errors.
-            ErrorKind::Uncategorized(_)
-            | ErrorKind::Storage(_)
-            | ErrorKind::ArrayError(_)
-            | ErrorKind::ExprError(_)
-            | ErrorKind::SerdeError(_)
-            | ErrorKind::SinkError(_)
-            | ErrorKind::RpcError(_)
-            | ErrorKind::AlignBarrier(_, _)
-            | ErrorKind::ConnectorError(_)
-            | ErrorKind::DmlError(_)
-            | ErrorKind::NotImplemented(_) => 999,
-        }
+/// A [`StreamError`] with a score, used to find the root cause of actor failures.
+#[derive(Debug, Clone)]
+struct ScoredStreamError {
+    error: StreamError,
+    score: Score,
+}
+
+impl std::fmt::Display for ScoredStreamError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        self.error.fmt(f)
     }
+}
 
-    fn stream_error_score(e: &StreamError) -> i32 {
-        use crate::error::ErrorKind;
-        match e.inner() {
-            // `UnexpectedExit` wraps the original error. Score on the inner error.
-            ErrorKind::UnexpectedExit { source, .. } => stream_error_score(source),
+impl std::error::Error for ScoredStreamError {
+    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
+        self.error.source()
+    }
 
-            // `BarrierSend` is likely to be caused by actor exit and not the root cause.
-            ErrorKind::BarrierSend { .. } => 1,
+    fn provide<'a>(&'a self, request: &mut std::error::Request<'a>) {
+        self.error.provide(request);
+        // HIGHLIGHT: Provide the score to make it retrievable from meta service.
+        request.provide_value(self.score);
+    }
+}
 
-            // Executor errors first.
-            ErrorKind::Executor(ee) => 2000 + stream_executor_error_score(ee),
+impl ScoredStreamError {
+    /// Score the given error based on hard-coded rules.
+    fn new(error: StreamError) -> Self {
+        // Explicitly list all error kinds here to notice developers to update this function when
+        // there are changes in error kinds.
+
+        fn stream_executor_error_score(e: &StreamExecutorError) -> i32 {
+            use crate::executor::error::ErrorKind;
+            match e.inner() {
+                // `ChannelClosed` or `ExchangeChannelClosed` is likely to be caused by actor exit
+                // and not the root cause.
+                ErrorKind::ChannelClosed(_) | ErrorKind::ExchangeChannelClosed(_) => 1,
+
+                // Normal errors.
+                ErrorKind::Uncategorized(_)
+                | ErrorKind::Storage(_)
+                | ErrorKind::ArrayError(_)
+                | ErrorKind::ExprError(_)
+                | ErrorKind::SerdeError(_)
+                | ErrorKind::SinkError(_, _)
+                | ErrorKind::RpcError(_)
+                | ErrorKind::AlignBarrier(_, _)
+                | ErrorKind::ConnectorError(_)
+                | ErrorKind::DmlError(_)
+                | ErrorKind::NotImplemented(_) => 999,
+            }
+        }
 
-            // Then other errors.
-            ErrorKind::Uncategorized(_)
-            | ErrorKind::Storage(_)
-            | ErrorKind::Expression(_)
-            | ErrorKind::Array(_)
-            | ErrorKind::Sink(_) => 1000,
+        fn stream_error_score(e: &StreamError) -> i32 {
+            use crate::error::ErrorKind;
+            match e.inner() {
+                // `UnexpectedExit` wraps the original error. Score on the inner error.
+                ErrorKind::UnexpectedExit { source, .. } => stream_error_score(source),
+
+                // `BarrierSend` is likely to be caused by actor exit and not the root cause.
+                ErrorKind::BarrierSend { .. } => 1,
+
+                // Executor errors first.
+                ErrorKind::Executor(ee) => 2000 + stream_executor_error_score(ee),
+
+                // Then other errors.
+                ErrorKind::Uncategorized(_)
+                | ErrorKind::Storage(_)
+                | ErrorKind::Expression(_)
+                | ErrorKind::Array(_)
+                | ErrorKind::Secret(_) => 1000,
+            }
         }
-    }
 
-    actor_errors
-        .into_iter()
-        .max_by_key(|&e| stream_error_score(e))
-        .cloned()
+        let score = Score(stream_error_score(&error));
+        Self { error, score }
+    }
 }
 
 #[cfg(test)]
 impl LocalBarrierManager {
-    pub(super) fn spawn_for_test() -> EventSender<LocalActorOperation> {
+    fn spawn_for_test() -> EventSender<LocalActorOperation> {
         use std::sync::atomic::AtomicU64;
         let (tx, rx) = unbounded_channel();
         let _join_handle = LocalBarrierWorker::spawn(
@@ -1011,3 +1139,86 @@ impl LocalBarrierManager {
         rx.await.unwrap()
     }
 }
+
+#[cfg(test)]
+pub(crate) mod barrier_test_utils {
+    use std::sync::Arc;
+
+    use assert_matches::assert_matches;
+    use futures::StreamExt;
+    use risingwave_pb::stream_service::streaming_control_stream_request::InitRequest;
+    use risingwave_pb::stream_service::{
+        streaming_control_stream_request, streaming_control_stream_response, InjectBarrierRequest,
+        StreamingControlStreamRequest, StreamingControlStreamResponse,
+    };
+    use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender};
+    use tokio_stream::wrappers::UnboundedReceiverStream;
+    use tonic::Status;
+
+    use crate::executor::Barrier;
+    use crate::task::barrier_manager::{ControlStreamHandle, EventSender, LocalActorOperation};
+    use crate::task::{ActorId, LocalBarrierManager, SharedContext};
+
+    pub(crate) struct LocalBarrierTestEnv {
+        pub shared_context: Arc<SharedContext>,
+        pub(super) actor_op_tx: EventSender<LocalActorOperation>,
+        pub request_tx: UnboundedSender<Result<StreamingControlStreamRequest, Status>>,
+        pub response_rx: UnboundedReceiver<Result<StreamingControlStreamResponse, Status>>,
+    }
+
+    impl LocalBarrierTestEnv {
+        pub(crate) async fn for_test() -> Self {
+            let actor_op_tx = LocalBarrierManager::spawn_for_test();
+
+            let (request_tx, request_rx) = unbounded_channel();
+            let (response_tx, mut response_rx) = unbounded_channel();
+
+            actor_op_tx.send_event(LocalActorOperation::NewControlStream {
+                handle: ControlStreamHandle::new(
+                    response_tx,
+                    UnboundedReceiverStream::new(request_rx).boxed(),
+                ),
+                init_request: InitRequest { prev_epoch: 0 },
+            });
+
+            assert_matches!(
+                response_rx.recv().await.unwrap().unwrap().response.unwrap(),
+                streaming_control_stream_response::Response::Init(_)
+            );
+
+            let shared_context = actor_op_tx
+                .send_and_await(LocalActorOperation::GetCurrentSharedContext)
+                .await
+                .unwrap();
+
+            Self {
+                shared_context,
+                actor_op_tx,
+                request_tx,
+                response_rx,
+            }
+        }
+
+        pub(crate) fn inject_barrier(
+            &self,
+            barrier: &Barrier,
+            actor_to_send: impl IntoIterator<Item = ActorId>,
+            actor_to_collect: impl IntoIterator<Item = ActorId>,
+        ) {
+            self.request_tx
+                .send(Ok(StreamingControlStreamRequest {
+                    request: Some(streaming_control_stream_request::Request::InjectBarrier(
+                        InjectBarrierRequest {
+                            request_id: "".to_string(),
+                            barrier: Some(barrier.to_protobuf()),
+                            actor_ids_to_send: actor_to_send.into_iter().collect(),
+                            actor_ids_to_collect: actor_to_collect.into_iter().collect(),
+                            table_ids_to_sync: vec![],
+                            partial_graph_id: u32::MAX,
+                        },
+                    )),
+                }))
+                .unwrap();
+        }
+    }
+}
diff --git a/src/stream/src/task/barrier_manager/managed_state.rs b/src/stream/src/task/barrier_manager/managed_state.rs
index 6c70525aa4b3a..3651dcc44d5e9 100644
--- a/src/stream/src/task/barrier_manager/managed_state.rs
+++ b/src/stream/src/task/barrier_manager/managed_state.rs
@@ -13,12 +13,12 @@
 // limitations under the License.
 
 use std::assert_matches::assert_matches;
-use std::collections::btree_map::Entry;
 use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
 use std::fmt::{Debug, Display, Formatter};
-use std::future::Future;
+use std::future::{poll_fn, Future};
 use std::mem::replace;
 use std::sync::Arc;
+use std::task::{ready, Context, Poll};
 
 use anyhow::anyhow;
 use await_tree::InstrumentAwait;
@@ -28,20 +28,20 @@ use futures::{FutureExt, StreamExt, TryFutureExt};
 use prometheus::HistogramTimer;
 use risingwave_common::catalog::TableId;
 use risingwave_common::must_match;
+use risingwave_common::util::epoch::EpochPair;
 use risingwave_hummock_sdk::SyncResult;
 use risingwave_pb::stream_plan::barrier::BarrierKind;
 use risingwave_pb::stream_service::barrier_complete_response::CreateMviewProgress;
 use risingwave_storage::{dispatch_state_store, StateStore, StateStoreImpl};
-use rw_futures_util::pending_on_none;
 use thiserror_ext::AsReport;
-use tokio::sync::oneshot;
+use tokio::sync::mpsc;
 
 use super::progress::BackfillState;
-use super::BarrierCompleteResult;
+use super::{BarrierCompleteResult, SubscribeMutationItem};
 use crate::error::StreamResult;
 use crate::executor::monitor::StreamingMetrics;
 use crate::executor::{Barrier, Mutation};
-use crate::task::{await_tree_key, ActorId};
+use crate::task::{await_tree_key, ActorId, PartialGraphId};
 
 struct IssuedState {
     pub mutation: Option<Arc<Mutation>>,
@@ -50,7 +50,8 @@ struct IssuedState {
 
     pub barrier_inflight_latency: HistogramTimer,
 
-    pub table_ids: HashSet<TableId>,
+    /// Only be `Some(_)` when `kind` is `Checkpoint`
+    pub table_ids: Option<HashSet<TableId>>,
 
     pub kind: BarrierKind,
 }
@@ -69,13 +70,6 @@ impl Debug for IssuedState {
 /// The state machine of local barrier manager.
 #[derive(Debug)]
 enum ManagedBarrierStateInner {
-    /// Received barrier from actors in other compute nodes in remote input, however no `send_barrier`
-    /// request from the meta service is issued.
-    Stashed {
-        /// Senders registered by the remote input.
-        mutation_senders: Vec<oneshot::Sender<Option<Arc<Mutation>>>>,
-    },
-
     /// Meta service has issued a `send_barrier` request. We're collecting barriers now.
     Issued(IssuedState),
 
@@ -120,22 +114,26 @@ fn sync_epoch<S: StateStore>(
         .boxed()
 }
 
-#[derive(Debug)]
 pub(super) struct ManagedBarrierStateDebugInfo<'a> {
-    epoch_barrier_state_map: &'a BTreeMap<u64, BarrierState>,
-
-    create_mview_progress: &'a HashMap<u64, HashMap<ActorId, BackfillState>>,
+    graph_states: &'a HashMap<PartialGraphId, PartialGraphManagedBarrierState>,
 }
 
 impl Display for ManagedBarrierStateDebugInfo<'_> {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        for (partial_graph_id, graph_states) in self.graph_states {
+            writeln!(f, "--- Partial Group {}", partial_graph_id.0)?;
+            write!(f, "{}", graph_states)?;
+        }
+        Ok(())
+    }
+}
+
+impl Display for &'_ PartialGraphManagedBarrierState {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
         let mut prev_epoch = 0u64;
-        for (epoch, barrier_state) in self.epoch_barrier_state_map {
+        for (epoch, barrier_state) in &self.epoch_barrier_state_map {
             write!(f, "> Epoch {}: ", epoch)?;
             match &barrier_state.inner {
-                ManagedBarrierStateInner::Stashed { .. } => {
-                    write!(f, "Stashed")?;
-                }
                 ManagedBarrierStateInner::Issued(state) => {
                     write!(f, "Issued [{:?}]. Remaining actors: [", state.kind)?;
                     let mut is_prev_epoch_issued = false;
@@ -181,7 +179,7 @@ impl Display for ManagedBarrierStateDebugInfo<'_> {
 
         if !self.create_mview_progress.is_empty() {
             writeln!(f, "Create MView Progress:")?;
-            for (epoch, progress) in self.create_mview_progress {
+            for (epoch, progress) in &self.create_mview_progress {
                 write!(f, "> Epoch {}:", epoch)?;
                 for (actor_id, state) in progress {
                     write!(f, ">> Actor {}: {}, ", actor_id, state)?;
@@ -193,12 +191,133 @@ impl Display for ManagedBarrierStateDebugInfo<'_> {
     }
 }
 
-pub(super) struct ManagedBarrierState {
+enum InflightActorStatus {
+    /// The actor is just spawned and not issued any barrier yet
+    NotStarted,
+    /// The actor has been issued some barriers, and not issued any stop barrier yet
+    Running(u64),
+    /// The actor has been issued a stop barrier
+    Stopping(u64),
+}
+
+impl InflightActorStatus {
+    pub(super) fn is_stopping(&self) -> bool {
+        matches!(self, InflightActorStatus::Stopping(_))
+    }
+
+    fn max_issued_epoch(&self) -> Option<u64> {
+        match self {
+            InflightActorStatus::NotStarted => None,
+            InflightActorStatus::Running(epoch) | InflightActorStatus::Stopping(epoch) => {
+                Some(*epoch)
+            }
+        }
+    }
+}
+
+pub(crate) struct InflightActorState {
+    pending_subscribers: BTreeMap<u64, Vec<mpsc::UnboundedSender<SubscribeMutationItem>>>,
+    /// `prev_epoch` -> partial graph id
+    pub(super) inflight_barriers: BTreeMap<u64, PartialGraphId>,
+    /// `prev_epoch` -> (`mutation`, `curr_epoch`)
+    barrier_mutations: BTreeMap<u64, (Option<Arc<Mutation>>, u64)>,
+    status: InflightActorStatus,
+}
+
+impl InflightActorState {
+    pub(super) fn not_started() -> Self {
+        Self {
+            pending_subscribers: Default::default(),
+            inflight_barriers: BTreeMap::default(),
+            barrier_mutations: Default::default(),
+            status: InflightActorStatus::NotStarted,
+        }
+    }
+
+    pub(super) fn issue_barrier(
+        &mut self,
+        partial_graph_id: PartialGraphId,
+        barrier: &Barrier,
+        is_stop: bool,
+    ) {
+        if let Some(max_issued_epoch) = self.status.max_issued_epoch() {
+            assert!(barrier.epoch.prev > max_issued_epoch);
+        }
+
+        if let Some((first_epoch, _)) = self.pending_subscribers.first_key_value() {
+            assert!(
+                *first_epoch >= barrier.epoch.prev,
+                "barrier epoch {:?} skip subscribed epoch {}",
+                barrier.epoch,
+                first_epoch
+            );
+            if *first_epoch == barrier.epoch.prev {
+                let (_, mut subscribers) = self.pending_subscribers.pop_first().expect("non empty");
+                subscribers.retain(|tx| {
+                    tx.send((barrier.epoch.prev, barrier.mutation.clone()))
+                        .is_ok()
+                });
+                if !is_stop && !subscribers.is_empty() {
+                    self.pending_subscribers
+                        .entry(barrier.epoch.curr)
+                        .or_default()
+                        .extend(subscribers);
+                }
+            }
+        }
+
+        assert!(self
+            .inflight_barriers
+            .insert(barrier.epoch.prev, partial_graph_id)
+            .is_none());
+
+        assert!(self
+            .barrier_mutations
+            .insert(
+                barrier.epoch.prev,
+                (barrier.mutation.clone(), barrier.epoch.curr),
+            )
+            .is_none());
+        if is_stop {
+            assert!(self.pending_subscribers.is_empty());
+            assert!(
+                !self.status.is_stopping(),
+                "stopped actor should not issue barrier"
+            );
+            self.status = InflightActorStatus::Stopping(barrier.epoch.prev);
+        } else {
+            self.status = InflightActorStatus::Running(barrier.epoch.prev);
+        }
+    }
+
+    pub(super) fn collect(&mut self, epoch: EpochPair) -> (PartialGraphId, bool) {
+        let (prev_epoch, prev_partial_graph_id) =
+            self.inflight_barriers.pop_first().expect("should exist");
+        assert_eq!(prev_epoch, epoch.prev);
+        let (min_mutation_epoch, _) = self.barrier_mutations.pop_first().expect("should exist");
+        assert_eq!(min_mutation_epoch, epoch.prev);
+        (
+            prev_partial_graph_id,
+            self.inflight_barriers.is_empty() && self.status.is_stopping(),
+        )
+    }
+
+    pub(super) fn is_running(&self) -> bool {
+        matches!(&self.status, InflightActorStatus::Running(_))
+    }
+}
+
+pub(super) struct PartialGraphManagedBarrierState {
+    /// This is a temporary workaround for the need to still calling `seal_epoch` for storage.
+    /// Can be removed after `seal_epoch` is deprecated in storage.
+    need_seal_epoch: bool,
     /// Record barrier state for each epoch of concurrent checkpoints.
     ///
     /// The key is `prev_epoch`, and the first value is `curr_epoch`
     epoch_barrier_state_map: BTreeMap<u64, BarrierState>,
 
+    prev_barrier_table_ids: Option<(EpochPair, HashSet<TableId>)>,
+
     /// Record the progress updates of creating mviews for each epoch of concurrent checkpoints.
     pub(super) create_mview_progress: HashMap<u64, HashMap<ActorId, BackfillState>>,
 
@@ -213,16 +332,54 @@ pub(super) struct ManagedBarrierState {
     barrier_await_tree_reg: Option<await_tree::Registry>,
 }
 
-impl ManagedBarrierState {
+impl PartialGraphManagedBarrierState {
+    fn new(
+        need_seal_epoch: bool,
+        state_store: StateStoreImpl,
+        streaming_metrics: Arc<StreamingMetrics>,
+        barrier_await_tree_reg: Option<await_tree::Registry>,
+    ) -> Self {
+        Self {
+            need_seal_epoch,
+            epoch_barrier_state_map: Default::default(),
+            prev_barrier_table_ids: None,
+            create_mview_progress: Default::default(),
+            await_epoch_completed_futures: Default::default(),
+            state_store,
+            streaming_metrics,
+            barrier_await_tree_reg,
+        }
+    }
+
     #[cfg(test)]
     pub(crate) fn for_test() -> Self {
         Self::new(
+            true,
             StateStoreImpl::for_test(),
             Arc::new(StreamingMetrics::unused()),
             None,
         )
     }
 
+    pub(super) fn is_empty(&self) -> bool {
+        self.epoch_barrier_state_map.is_empty()
+    }
+}
+
+pub(super) struct ManagedBarrierState {
+    pub(super) actor_states: HashMap<ActorId, InflightActorState>,
+
+    pub(super) graph_states: HashMap<PartialGraphId, PartialGraphManagedBarrierState>,
+
+    pub(super) state_store: StateStoreImpl,
+
+    pub(super) streaming_metrics: Arc<StreamingMetrics>,
+
+    /// Manages the await-trees of all barriers.
+    barrier_await_tree_reg: Option<await_tree::Registry>,
+}
+
+impl ManagedBarrierState {
     /// Create a barrier manager state. This will be called only once.
     pub(super) fn new(
         state_store: StateStoreImpl,
@@ -230,58 +387,152 @@ impl ManagedBarrierState {
         barrier_await_tree_reg: Option<await_tree::Registry>,
     ) -> Self {
         Self {
-            epoch_barrier_state_map: BTreeMap::default(),
-            create_mview_progress: Default::default(),
+            actor_states: Default::default(),
+            graph_states: Default::default(),
             state_store,
             streaming_metrics,
-            await_epoch_completed_futures: FuturesOrdered::new(),
             barrier_await_tree_reg,
         }
     }
 
     pub(super) fn to_debug_info(&self) -> ManagedBarrierStateDebugInfo<'_> {
         ManagedBarrierStateDebugInfo {
-            epoch_barrier_state_map: &self.epoch_barrier_state_map,
-            create_mview_progress: &self.create_mview_progress,
+            graph_states: &self.graph_states,
+        }
+    }
+}
+
+impl InflightActorState {
+    pub(super) fn subscribe_actor_mutation(
+        &mut self,
+        start_prev_epoch: u64,
+        tx: mpsc::UnboundedSender<SubscribeMutationItem>,
+    ) {
+        if let Some((mutation, start_curr_epoch)) = self.barrier_mutations.get(&start_prev_epoch) {
+            if tx.send((start_prev_epoch, mutation.clone())).is_err() {
+                return;
+            }
+            let mut prev_epoch = *start_curr_epoch;
+            for (mutation_prev_epoch, (mutation, mutation_curr_epoch)) in
+                self.barrier_mutations.range(start_curr_epoch..)
+            {
+                assert_eq!(prev_epoch, *mutation_prev_epoch);
+                if tx.send((prev_epoch, mutation.clone())).is_err() {
+                    // No more subscribe on the mutation. Simply return.
+                    return;
+                }
+                prev_epoch = *mutation_curr_epoch;
+            }
+            if !self.status.is_stopping() {
+                // Only add the subscribers when the actor is not stopped yet.
+                self.pending_subscribers
+                    .entry(prev_epoch)
+                    .or_default()
+                    .push(tx);
+            }
+        } else {
+            // Barrier has not issued yet. Store the pending tx
+            if let Some(max_issued_epoch) = self.status.max_issued_epoch() {
+                assert!(
+                    max_issued_epoch < start_prev_epoch,
+                    "later barrier {} has been issued, but skip the start epoch {:?}",
+                    max_issued_epoch,
+                    start_prev_epoch
+                );
+            } else {
+                assert!(!self.status.is_stopping(), "actor has been stopped and has not inflight barrier. unlikely to get further barrier");
+            }
+            self.pending_subscribers
+                .entry(start_prev_epoch)
+                .or_default()
+                .push(tx);
         }
     }
+}
+
+impl ManagedBarrierState {
+    pub(super) fn subscribe_actor_mutation(
+        &mut self,
+        actor_id: ActorId,
+        start_prev_epoch: u64,
+        tx: mpsc::UnboundedSender<SubscribeMutationItem>,
+    ) {
+        self.actor_states
+            .entry(actor_id)
+            .or_insert_with(InflightActorState::not_started)
+            .subscribe_actor_mutation(start_prev_epoch, tx);
+    }
 
-    pub fn read_barrier_mutation(
+    pub(super) fn transform_to_issued(
         &mut self,
         barrier: &Barrier,
-        sender: oneshot::Sender<Option<Arc<Mutation>>>,
+        actor_ids_to_collect: HashSet<ActorId>,
+        table_ids: HashSet<TableId>,
+        partial_graph_id: PartialGraphId,
     ) {
-        match self.epoch_barrier_state_map.entry(barrier.epoch.prev) {
-            Entry::Vacant(v) => {
-                v.insert(BarrierState {
-                    curr_epoch: barrier.epoch.curr,
-                    inner: ManagedBarrierStateInner::Stashed {
-                        mutation_senders: vec![sender],
-                    },
-                });
-            }
-            Entry::Occupied(mut o) => {
-                let state = o.get_mut();
-                match &mut state.inner {
-                    ManagedBarrierStateInner::Stashed {
-                        ref mut mutation_senders,
-                    } => {
-                        mutation_senders.push(sender);
-                    }
-                    ManagedBarrierStateInner::Issued(IssuedState { mutation, .. }) => {
-                        let _ = sender.send(mutation.clone());
-                    }
-                    _ => {
-                        panic!(
-                            "cannot read barrier mutation {:?} at current state: {:?}",
-                            barrier.epoch, state.inner
-                        )
-                    }
+        let actor_to_stop = barrier.all_stop_actors();
+        let graph_state = self
+            .graph_states
+            .entry(partial_graph_id)
+            .or_insert_with(|| {
+                PartialGraphManagedBarrierState::new(
+                    partial_graph_id.is_global_graph(),
+                    self.state_store.clone(),
+                    self.streaming_metrics.clone(),
+                    self.barrier_await_tree_reg.clone(),
+                )
+            });
+
+        graph_state.transform_to_issued(barrier, actor_ids_to_collect.clone(), table_ids);
+
+        // Note: it's important to issue barrier to actor after issuing to graph to ensure that
+        // we call `start_epoch` on the graph before the actors receive the barrier
+        for actor_id in actor_ids_to_collect {
+            self.actor_states
+                .entry(actor_id)
+                .or_insert_with(InflightActorState::not_started)
+                .issue_barrier(
+                    partial_graph_id,
+                    barrier,
+                    actor_to_stop
+                        .map(|actors| actors.contains(&actor_id))
+                        .unwrap_or(false),
+                );
+        }
+    }
+
+    pub(super) fn next_completed_epoch(
+        &mut self,
+    ) -> impl Future<Output = (PartialGraphId, u64)> + '_ {
+        poll_fn(|cx| {
+            for (partial_graph_id, graph_state) in &mut self.graph_states {
+                if let Poll::Ready(epoch) = graph_state.poll_next_completed_epoch(cx) {
+                    let partial_graph_id = *partial_graph_id;
+                    return Poll::Ready((partial_graph_id, epoch));
                 }
             }
+            Poll::Pending
+        })
+    }
+
+    pub(super) fn collect(&mut self, actor_id: ActorId, epoch: EpochPair) {
+        let (prev_partial_graph_id, is_finished) = self
+            .actor_states
+            .get_mut(&actor_id)
+            .expect("should exist")
+            .collect(epoch);
+        if is_finished {
+            self.actor_states.remove(&actor_id);
         }
+        let prev_graph_state = self
+            .graph_states
+            .get_mut(&prev_partial_graph_id)
+            .expect("should exist");
+        prev_graph_state.collect(actor_id, epoch);
     }
+}
 
+impl PartialGraphManagedBarrierState {
     /// This method is called when barrier state is modified in either `Issued` or `Stashed`
     /// to transform the state to `AllCollected` and start state store `sync` when the barrier
     /// has been collected from all actors for an `Issued` barrier.
@@ -300,7 +551,7 @@ impl ManagedBarrierState {
                 ManagedBarrierStateInner::AllCollected | ManagedBarrierStateInner::Completed(_) => {
                     continue;
                 }
-                ManagedBarrierStateInner::Stashed { .. } | ManagedBarrierStateInner::Issued(_) => {
+                ManagedBarrierStateInner::Issued(_) => {
                     break;
                 }
             }
@@ -357,19 +608,23 @@ impl ManagedBarrierState {
                     None
                 }
                 BarrierKind::Barrier => {
-                    dispatch_state_store!(&self.state_store, state_store, {
-                        state_store.seal_epoch(prev_epoch, kind.is_checkpoint());
-                    });
+                    if self.need_seal_epoch {
+                        dispatch_state_store!(&self.state_store, state_store, {
+                            state_store.seal_epoch(prev_epoch, kind.is_checkpoint());
+                        });
+                    }
                     None
                 }
                 BarrierKind::Checkpoint => {
                     dispatch_state_store!(&self.state_store, state_store, {
-                        state_store.seal_epoch(prev_epoch, kind.is_checkpoint());
+                        if self.need_seal_epoch {
+                            state_store.seal_epoch(prev_epoch, kind.is_checkpoint());
+                        }
                         Some(sync_epoch(
                             state_store,
                             &self.streaming_metrics,
                             prev_epoch,
-                            table_ids,
+                            table_ids.expect("should be Some on BarrierKind::Checkpoint"),
                         ))
                     })
                 }
@@ -407,55 +662,22 @@ impl ManagedBarrierState {
         }
     }
 
-    /// Returns an iterator on epochs that is awaiting on `actor_id`.
-    /// This is used on notifying actor failure. On actor failure, the
-    /// barrier manager can call this method to iterate on epochs that
-    /// waits on the failed actor and then notify failure on the result
-    /// sender of the epoch.
-    pub(crate) fn epochs_await_on_actor(
-        &self,
-        actor_id: ActorId,
-    ) -> impl Iterator<Item = u64> + '_ {
-        self.epoch_barrier_state_map
-            .iter()
-            .filter_map(move |(prev_epoch, barrier_state)| {
-                #[allow(clippy::single_match)]
-                match barrier_state.inner {
-                    ManagedBarrierStateInner::Issued(IssuedState {
-                        ref remaining_actors,
-                        ..
-                    }) => {
-                        if remaining_actors.contains(&actor_id) {
-                            Some(*prev_epoch)
-                        } else {
-                            None
-                        }
-                    }
-                    _ => None,
-                }
-            })
-    }
-
     /// Collect a `barrier` from the actor with `actor_id`.
-    pub(super) fn collect(&mut self, actor_id: ActorId, barrier: &Barrier) {
+    pub(super) fn collect(&mut self, actor_id: ActorId, epoch: EpochPair) {
         tracing::debug!(
             target: "events::stream::barrier::manager::collect",
-            epoch = ?barrier.epoch, actor_id, state = ?self.epoch_barrier_state_map,
+            ?epoch, actor_id, state = ?self.epoch_barrier_state_map,
             "collect_barrier",
         );
 
-        match self.epoch_barrier_state_map.get_mut(&barrier.epoch.prev) {
-            Some(&mut BarrierState {
-                inner: ManagedBarrierStateInner::Stashed { .. },
-                ..
-            })
-            | None => {
+        match self.epoch_barrier_state_map.get_mut(&epoch.prev) {
+            None => {
                 // If the barrier's state is stashed, this occurs exclusively in scenarios where the barrier has not been
                 // injected by the barrier manager, or the barrier message is blocked at the `RemoteInput` side waiting for injection.
                 // Given these conditions, it's inconceivable for an actor to attempt collect at this point.
                 panic!(
-                    "cannot collect new actor barrier {:?} at current state: Stashed or None",
-                    barrier.epoch,
+                    "cannot collect new actor barrier {:?} at current state: None",
+                    epoch,
                 )
             }
             Some(&mut BarrierState {
@@ -471,15 +693,15 @@ impl ManagedBarrierState {
                 assert!(
                     exist,
                     "the actor doesn't exist. actor_id: {:?}, curr_epoch: {:?}",
-                    actor_id, barrier.epoch.curr
+                    actor_id, epoch.curr
                 );
-                assert_eq!(curr_epoch, barrier.epoch.curr);
-                self.may_have_collected_all(barrier.epoch.prev);
+                assert_eq!(curr_epoch, epoch.curr);
+                self.may_have_collected_all(epoch.prev);
             }
             Some(BarrierState { inner, .. }) => {
                 panic!(
                     "cannot collect new actor barrier {:?} at current state: {:?}",
-                    barrier.epoch, inner
+                    epoch, inner
                 )
             }
         }
@@ -497,26 +719,61 @@ impl ManagedBarrierState {
             .streaming_metrics
             .barrier_inflight_latency
             .start_timer();
-        match self.epoch_barrier_state_map.get_mut(&barrier.epoch.prev) {
-            Some(&mut BarrierState {
-                inner:
-                    ManagedBarrierStateInner::Stashed {
-                        ref mut mutation_senders,
-                    },
-                ..
-            }) => {
-                for sender in mutation_senders.drain(..) {
-                    let _ = sender.send(barrier.mutation.clone());
+
+        if let Some(hummock) = self.state_store.as_hummock() {
+            hummock.start_epoch(barrier.epoch.curr, table_ids.clone());
+        }
+
+        let table_ids = match barrier.kind {
+            BarrierKind::Unspecified => {
+                unreachable!()
+            }
+            BarrierKind::Initial => {
+                assert!(
+                    self.prev_barrier_table_ids.is_none(),
+                    "non empty table_ids at initial barrier: {:?}",
+                    self.prev_barrier_table_ids
+                );
+                info!(epoch = ?barrier.epoch, "initialize at Initial barrier");
+                self.prev_barrier_table_ids = Some((barrier.epoch, table_ids));
+                None
+            }
+            BarrierKind::Barrier => {
+                if let Some((prev_epoch, prev_table_ids)) = self.prev_barrier_table_ids.as_mut() {
+                    assert_eq!(prev_epoch.curr, barrier.epoch.prev);
+                    assert_eq!(prev_table_ids, &table_ids);
+                    *prev_epoch = barrier.epoch;
+                } else {
+                    info!(epoch = ?barrier.epoch, "initialize at non-checkpoint barrier");
+                    self.prev_barrier_table_ids = Some((barrier.epoch, table_ids));
                 }
+                None
             }
-            Some(BarrierState { ref inner, .. }) => {
+            BarrierKind::Checkpoint => Some(
+                if let Some((prev_epoch, prev_table_ids)) = self
+                    .prev_barrier_table_ids
+                    .replace((barrier.epoch, table_ids))
+                    && prev_epoch.curr == barrier.epoch.prev
+                {
+                    prev_table_ids
+                } else {
+                    debug!(epoch = ?barrier.epoch, "reinitialize at Checkpoint barrier");
+                    HashSet::new()
+                },
+            ),
+        };
+
+        if let Some(BarrierState { ref inner, .. }) =
+            self.epoch_barrier_state_map.get_mut(&barrier.epoch.prev)
+        {
+            {
                 panic!(
                     "barrier epochs{:?} state has already been `Issued`. Current state: {:?}",
                     barrier.epoch, inner
                 );
             }
-            None => {}
         };
+
         self.epoch_barrier_state_map.insert(
             barrier.epoch.prev,
             BarrierState {
@@ -534,17 +791,20 @@ impl ManagedBarrierState {
     }
 
     /// Return a future that yields the next completed epoch. The future is cancellation safe.
-    pub(crate) fn next_completed_epoch(&mut self) -> impl Future<Output = u64> + '_ {
-        pending_on_none(self.await_epoch_completed_futures.next()).map(|(prev_epoch, result)| {
-            let state = self
-                .epoch_barrier_state_map
-                .get_mut(&prev_epoch)
-                .expect("should exist");
-            // sanity check on barrier state
-            assert_matches!(&state.inner, ManagedBarrierStateInner::AllCollected);
-            state.inner = ManagedBarrierStateInner::Completed(result);
-            prev_epoch
-        })
+    pub(crate) fn poll_next_completed_epoch(&mut self, cx: &mut Context<'_>) -> Poll<u64> {
+        ready!(self.await_epoch_completed_futures.next().poll_unpin(cx))
+            .map(|(prev_epoch, result)| {
+                let state = self
+                    .epoch_barrier_state_map
+                    .get_mut(&prev_epoch)
+                    .expect("should exist");
+                // sanity check on barrier state
+                assert_matches!(&state.inner, ManagedBarrierStateInner::AllCollected);
+                state.inner = ManagedBarrierStateInner::Completed(result);
+                prev_epoch
+            })
+            .map(Poll::Ready)
+            .unwrap_or(Poll::Pending)
     }
 
     /// Pop the completion result of an completed epoch.
@@ -587,7 +847,7 @@ impl ManagedBarrierState {
 
     #[cfg(test)]
     async fn pop_next_completed_epoch(&mut self) -> u64 {
-        let epoch = self.next_completed_epoch().await;
+        let epoch = poll_fn(|cx| self.poll_next_completed_epoch(cx)).await;
         let _ = self.pop_completed_epoch(epoch).unwrap().unwrap();
         epoch
     }
@@ -600,11 +860,11 @@ mod tests {
     use risingwave_common::util::epoch::test_epoch;
 
     use crate::executor::Barrier;
-    use crate::task::barrier_manager::managed_state::ManagedBarrierState;
+    use crate::task::barrier_manager::managed_state::PartialGraphManagedBarrierState;
 
     #[tokio::test]
     async fn test_managed_state_add_actor() {
-        let mut managed_barrier_state = ManagedBarrierState::for_test();
+        let mut managed_barrier_state = PartialGraphManagedBarrierState::for_test();
         let barrier1 = Barrier::new_test_barrier(test_epoch(1));
         let barrier2 = Barrier::new_test_barrier(test_epoch(2));
         let barrier3 = Barrier::new_test_barrier(test_epoch(3));
@@ -614,8 +874,8 @@ mod tests {
         managed_barrier_state.transform_to_issued(&barrier1, actor_ids_to_collect1, HashSet::new());
         managed_barrier_state.transform_to_issued(&barrier2, actor_ids_to_collect2, HashSet::new());
         managed_barrier_state.transform_to_issued(&barrier3, actor_ids_to_collect3, HashSet::new());
-        managed_barrier_state.collect(1, &barrier1);
-        managed_barrier_state.collect(2, &barrier1);
+        managed_barrier_state.collect(1, barrier1.epoch);
+        managed_barrier_state.collect(2, barrier1.epoch);
         assert_eq!(
             managed_barrier_state.pop_next_completed_epoch().await,
             test_epoch(0)
@@ -628,9 +888,9 @@ mod tests {
                 .0,
             &test_epoch(1)
         );
-        managed_barrier_state.collect(1, &barrier2);
-        managed_barrier_state.collect(1, &barrier3);
-        managed_barrier_state.collect(2, &barrier2);
+        managed_barrier_state.collect(1, barrier2.epoch);
+        managed_barrier_state.collect(1, barrier3.epoch);
+        managed_barrier_state.collect(2, barrier2.epoch);
         assert_eq!(
             managed_barrier_state.pop_next_completed_epoch().await,
             test_epoch(1)
@@ -643,8 +903,8 @@ mod tests {
                 .0,
             { &test_epoch(2) }
         );
-        managed_barrier_state.collect(2, &barrier3);
-        managed_barrier_state.collect(3, &barrier3);
+        managed_barrier_state.collect(2, barrier3.epoch);
+        managed_barrier_state.collect(3, barrier3.epoch);
         assert_eq!(
             managed_barrier_state.pop_next_completed_epoch().await,
             test_epoch(2)
@@ -654,7 +914,7 @@ mod tests {
 
     #[tokio::test]
     async fn test_managed_state_stop_actor() {
-        let mut managed_barrier_state = ManagedBarrierState::for_test();
+        let mut managed_barrier_state = PartialGraphManagedBarrierState::for_test();
         let barrier1 = Barrier::new_test_barrier(test_epoch(1));
         let barrier2 = Barrier::new_test_barrier(test_epoch(2));
         let barrier3 = Barrier::new_test_barrier(test_epoch(3));
@@ -665,12 +925,12 @@ mod tests {
         managed_barrier_state.transform_to_issued(&barrier2, actor_ids_to_collect2, HashSet::new());
         managed_barrier_state.transform_to_issued(&barrier3, actor_ids_to_collect3, HashSet::new());
 
-        managed_barrier_state.collect(1, &barrier1);
-        managed_barrier_state.collect(1, &barrier2);
-        managed_barrier_state.collect(1, &barrier3);
-        managed_barrier_state.collect(2, &barrier1);
-        managed_barrier_state.collect(2, &barrier2);
-        managed_barrier_state.collect(2, &barrier3);
+        managed_barrier_state.collect(1, barrier1.epoch);
+        managed_barrier_state.collect(1, barrier2.epoch);
+        managed_barrier_state.collect(1, barrier3.epoch);
+        managed_barrier_state.collect(2, barrier1.epoch);
+        managed_barrier_state.collect(2, barrier2.epoch);
+        managed_barrier_state.collect(2, barrier3.epoch);
         assert_eq!(
             managed_barrier_state
                 .epoch_barrier_state_map
@@ -679,8 +939,8 @@ mod tests {
                 .0,
             &0
         );
-        managed_barrier_state.collect(3, &barrier1);
-        managed_barrier_state.collect(3, &barrier2);
+        managed_barrier_state.collect(3, barrier1.epoch);
+        managed_barrier_state.collect(3, barrier2.epoch);
         assert_eq!(
             managed_barrier_state
                 .epoch_barrier_state_map
@@ -689,7 +949,7 @@ mod tests {
                 .0,
             &0
         );
-        managed_barrier_state.collect(4, &barrier1);
+        managed_barrier_state.collect(4, barrier1.epoch);
         assert_eq!(
             managed_barrier_state.pop_next_completed_epoch().await,
             test_epoch(0)
diff --git a/src/stream/src/task/barrier_manager/progress.rs b/src/stream/src/task/barrier_manager/progress.rs
index 69c603a4b1ab4..9a243c2e975d1 100644
--- a/src/stream/src/task/barrier_manager/progress.rs
+++ b/src/stream/src/task/barrier_manager/progress.rs
@@ -14,6 +14,8 @@
 
 use std::fmt::{Display, Formatter};
 
+use risingwave_common::util::epoch::EpochPair;
+
 use super::LocalBarrierManager;
 use crate::task::barrier_manager::LocalBarrierEvent::ReportCreateProgress;
 use crate::task::barrier_manager::LocalBarrierWorker;
@@ -42,27 +44,29 @@ impl Display for BackfillState {
 impl LocalBarrierWorker {
     pub(crate) fn update_create_mview_progress(
         &mut self,
-        current_epoch: u64,
+        epoch: EpochPair,
         actor: ActorId,
         state: BackfillState,
     ) {
-        self.state
-            .create_mview_progress
-            .entry(current_epoch)
-            .or_default()
-            .insert(actor, state);
+        if let Some(actor_state) = self.state.actor_states.get(&actor)
+            && let Some(partial_graph_id) = actor_state.inflight_barriers.get(&epoch.prev)
+            && let Some(graph_state) = self.state.graph_states.get_mut(partial_graph_id)
+        {
+            graph_state
+                .create_mview_progress
+                .entry(epoch.curr)
+                .or_default()
+                .insert(actor, state);
+        } else {
+            warn!(?epoch, actor, ?state, "ignore create mview progress");
+        }
     }
 }
 
 impl LocalBarrierManager {
-    fn update_create_mview_progress(
-        &self,
-        current_epoch: u64,
-        actor: ActorId,
-        state: BackfillState,
-    ) {
+    fn update_create_mview_progress(&self, epoch: EpochPair, actor: ActorId, state: BackfillState) {
         self.send_event(ReportCreateProgress {
-            current_epoch,
+            epoch,
             actor,
             state,
         })
@@ -126,13 +130,10 @@ impl CreateMviewProgress {
         self.backfill_actor_id
     }
 
-    fn update_inner(&mut self, current_epoch: u64, state: BackfillState) {
+    fn update_inner(&mut self, epoch: EpochPair, state: BackfillState) {
         self.state = Some(state);
-        self.barrier_manager.update_create_mview_progress(
-            current_epoch,
-            self.backfill_actor_id,
-            state,
-        );
+        self.barrier_manager
+            .update_create_mview_progress(epoch, self.backfill_actor_id, state);
     }
 
     /// Update the progress to `ConsumingUpstream(consumed_epoch, consumed_rows)`. The epoch must be
@@ -141,7 +142,7 @@ impl CreateMviewProgress {
     /// `current_consumed_rows` is an accumulated value.
     pub fn update(
         &mut self,
-        current_epoch: u64,
+        epoch: EpochPair,
         consumed_epoch: ConsumedEpoch,
         current_consumed_rows: ConsumedRows,
     ) {
@@ -159,18 +160,18 @@ impl CreateMviewProgress {
             None => {}
         };
         self.update_inner(
-            current_epoch,
+            epoch,
             BackfillState::ConsumingUpstream(consumed_epoch, current_consumed_rows),
         );
     }
 
     /// Finish the progress. If the progress is already finished, then perform no-op.
     /// `current_epoch` should be provided to locate the barrier under concurrent checkpoint.
-    pub fn finish(&mut self, current_epoch: u64, current_consumed_rows: ConsumedRows) {
+    pub fn finish(&mut self, epoch: EpochPair, current_consumed_rows: ConsumedRows) {
         if let Some(BackfillState::Done(_)) = self.state {
             return;
         }
-        self.update_inner(current_epoch, BackfillState::Done(current_consumed_rows));
+        self.update_inner(epoch, BackfillState::Done(current_consumed_rows));
     }
 }
 
diff --git a/src/stream/src/task/barrier_manager/tests.rs b/src/stream/src/task/barrier_manager/tests.rs
index 82a08e1d66117..0d1bb159ea5f7 100644
--- a/src/stream/src/task/barrier_manager/tests.rs
+++ b/src/stream/src/task/barrier_manager/tests.rs
@@ -17,44 +17,21 @@ use std::iter::once;
 use std::pin::pin;
 use std::task::Poll;
 
-use assert_matches::assert_matches;
 use futures::future::join_all;
 use futures::FutureExt;
 use risingwave_common::util::epoch::test_epoch;
-use risingwave_pb::stream_service::{streaming_control_stream_request, InjectBarrierRequest};
-use tokio_stream::wrappers::UnboundedReceiverStream;
 
 use super::*;
+use crate::task::barrier_test_utils::LocalBarrierTestEnv;
 
 #[tokio::test]
 async fn test_managed_barrier_collection() -> StreamResult<()> {
-    let actor_op_tx = LocalBarrierManager::spawn_for_test();
+    let mut test_env = LocalBarrierTestEnv::for_test().await;
 
-    let (request_tx, request_rx) = unbounded_channel();
-    let (response_tx, mut response_rx) = unbounded_channel();
-
-    actor_op_tx.send_event(LocalActorOperation::NewControlStream {
-        handle: ControlStreamHandle::new(
-            response_tx,
-            UnboundedReceiverStream::new(request_rx).boxed(),
-        ),
-        init_request: InitRequest { prev_epoch: 0 },
-    });
-
-    assert_matches!(
-        response_rx.recv().await.unwrap().unwrap().response.unwrap(),
-        streaming_control_stream_response::Response::Init(_)
-    );
-
-    let context = actor_op_tx
-        .send_and_await(LocalActorOperation::GetCurrentSharedContext)
-        .await
-        .unwrap();
-
-    let manager = &context.local_barrier_manager;
+    let manager = &test_env.shared_context.local_barrier_manager;
 
     let register_sender = |actor_id: u32| {
-        let actor_op_tx = &actor_op_tx;
+        let actor_op_tx = &test_env.actor_op_tx;
         async move {
             let (barrier_tx, barrier_rx) = unbounded_channel();
             actor_op_tx
@@ -79,19 +56,7 @@ async fn test_managed_barrier_collection() -> StreamResult<()> {
     let barrier = Barrier::new_test_barrier(curr_epoch);
     let epoch = barrier.epoch.prev;
 
-    request_tx
-        .send(Ok(StreamingControlStreamRequest {
-            request: Some(streaming_control_stream_request::Request::InjectBarrier(
-                InjectBarrierRequest {
-                    request_id: "".to_string(),
-                    barrier: Some(barrier.to_protobuf()),
-                    actor_ids_to_send: actor_ids.clone(),
-                    actor_ids_to_collect: actor_ids,
-                    table_ids_to_sync: vec![],
-                },
-            )),
-        }))
-        .unwrap();
+    test_env.inject_barrier(&barrier, actor_ids.clone(), actor_ids);
 
     // Collect barriers from actors
     let collected_barriers = join_all(rxs.iter_mut().map(|(actor_id, rx)| async move {
@@ -101,7 +66,7 @@ async fn test_managed_barrier_collection() -> StreamResult<()> {
     }))
     .await;
 
-    let mut await_epoch_future = pin!(response_rx.recv().map(|result| {
+    let mut await_epoch_future = pin!(test_env.response_rx.recv().map(|result| {
         let resp: StreamingControlStreamResponse = result.unwrap().unwrap();
         let resp = resp.response.unwrap();
         match resp {
@@ -124,33 +89,12 @@ async fn test_managed_barrier_collection() -> StreamResult<()> {
 
 #[tokio::test]
 async fn test_managed_barrier_collection_separately() -> StreamResult<()> {
-    let actor_op_tx = LocalBarrierManager::spawn_for_test();
-
-    let (request_tx, request_rx) = unbounded_channel();
-    let (response_tx, mut response_rx) = unbounded_channel();
-
-    actor_op_tx.send_event(LocalActorOperation::NewControlStream {
-        handle: ControlStreamHandle::new(
-            response_tx,
-            UnboundedReceiverStream::new(request_rx).boxed(),
-        ),
-        init_request: InitRequest { prev_epoch: 0 },
-    });
+    let mut test_env = LocalBarrierTestEnv::for_test().await;
 
-    assert_matches!(
-        response_rx.recv().await.unwrap().unwrap().response.unwrap(),
-        streaming_control_stream_response::Response::Init(_)
-    );
-
-    let context = actor_op_tx
-        .send_and_await(LocalActorOperation::GetCurrentSharedContext)
-        .await
-        .unwrap();
-
-    let manager = &context.local_barrier_manager;
+    let manager = &test_env.shared_context.local_barrier_manager;
 
     let register_sender = |actor_id: u32| {
-        let actor_op_tx = &actor_op_tx;
+        let actor_op_tx = &test_env.actor_op_tx;
         async move {
             let (barrier_tx, barrier_rx) = unbounded_channel();
             actor_op_tx
@@ -179,29 +123,19 @@ async fn test_managed_barrier_collection_separately() -> StreamResult<()> {
 
     // Prepare the barrier
     let curr_epoch = test_epoch(2);
-    let barrier = Barrier::new_test_barrier(curr_epoch);
-    let epoch = barrier.epoch.prev;
+    let barrier = Barrier::new_test_barrier(curr_epoch).with_stop();
+
+    let mut mutation_subscriber =
+        manager.subscribe_barrier_mutation(extra_actor_id, &barrier.clone().into_dispatcher());
 
     // Read the mutation after receiving the barrier from remote input.
-    let mut mutation_reader = pin!(manager.read_barrier_mutation(&barrier));
+    let mut mutation_reader = pin!(mutation_subscriber.recv());
     assert!(poll_fn(|cx| Poll::Ready(mutation_reader.as_mut().poll(cx).is_pending())).await);
 
-    request_tx
-        .send(Ok(StreamingControlStreamRequest {
-            request: Some(streaming_control_stream_request::Request::InjectBarrier(
-                InjectBarrierRequest {
-                    request_id: "".to_string(),
-                    barrier: Some(barrier.to_protobuf()),
-                    actor_ids_to_send,
-                    actor_ids_to_collect,
-                    table_ids_to_sync: vec![],
-                },
-            )),
-        }))
-        .unwrap();
-
-    let mutation = mutation_reader.await.unwrap();
-    assert_eq!(mutation, barrier.mutation);
+    test_env.inject_barrier(&barrier, actor_ids_to_send, actor_ids_to_collect);
+
+    let (epoch, mutation) = mutation_reader.await.unwrap();
+    assert_eq!((epoch, &mutation), (barrier.epoch.prev, &barrier.mutation));
 
     // Collect a barrier before sending
     manager.collect(extra_actor_id, &barrier);
@@ -214,7 +148,7 @@ async fn test_managed_barrier_collection_separately() -> StreamResult<()> {
     }))
     .await;
 
-    let mut await_epoch_future = pin!(response_rx.recv().map(|result| {
+    let mut await_epoch_future = pin!(test_env.response_rx.recv().map(|result| {
         let resp: StreamingControlStreamResponse = result.unwrap().unwrap();
         let resp = resp.response.unwrap();
         match resp {
diff --git a/src/stream/src/task/env.rs b/src/stream/src/task/env.rs
index a47eb8279224c..ce446b0d1bc68 100644
--- a/src/stream/src/task/env.rs
+++ b/src/stream/src/task/env.rs
@@ -20,7 +20,7 @@ use risingwave_common::system_param::local_manager::LocalSystemParamsManagerRef;
 use risingwave_common::util::addr::HostAddr;
 use risingwave_connector::source::monitor::SourceMetrics;
 use risingwave_dml::dml_manager::DmlManagerRef;
-use risingwave_rpc_client::MetaClient;
+use risingwave_rpc_client::{ComputeClientPoolRef, MetaClient};
 use risingwave_storage::StateStoreImpl;
 
 pub(crate) type WorkerNodeId = u32;
@@ -55,6 +55,9 @@ pub struct StreamEnvironment {
 
     /// Meta client. Use `None` for test only
     meta_client: Option<MetaClient>,
+
+    /// Compute client pool for streaming gRPC exchange.
+    client_pool: ComputeClientPoolRef,
 }
 
 impl StreamEnvironment {
@@ -68,6 +71,7 @@ impl StreamEnvironment {
         system_params_manager: LocalSystemParamsManagerRef,
         source_metrics: Arc<SourceMetrics>,
         meta_client: MetaClient,
+        client_pool: ComputeClientPoolRef,
     ) -> Self {
         StreamEnvironment {
             server_addr,
@@ -79,6 +83,7 @@ impl StreamEnvironment {
             source_metrics,
             total_mem_val: Arc::new(TrAdder::new()),
             meta_client: Some(meta_client),
+            client_pool,
         }
     }
 
@@ -87,9 +92,10 @@ impl StreamEnvironment {
     pub fn for_test() -> Self {
         use risingwave_common::system_param::local_manager::LocalSystemParamsManager;
         use risingwave_dml::dml_manager::DmlManager;
+        use risingwave_rpc_client::ComputeClientPool;
         use risingwave_storage::monitor::MonitoredStorageMetrics;
         StreamEnvironment {
-            server_addr: "127.0.0.1:5688".parse().unwrap(),
+            server_addr: "127.0.0.1:2333".parse().unwrap(),
             config: Arc::new(StreamingConfig::default()),
             worker_id: WorkerNodeId::default(),
             state_store: StateStoreImpl::shared_in_memory_store(Arc::new(
@@ -100,6 +106,7 @@ impl StreamEnvironment {
             source_metrics: Arc::new(SourceMetrics::default()),
             total_mem_val: Arc::new(TrAdder::new()),
             meta_client: None,
+            client_pool: Arc::new(ComputeClientPool::for_test()),
         }
     }
 
@@ -138,4 +145,8 @@ impl StreamEnvironment {
     pub fn meta_client(&self) -> Option<MetaClient> {
         self.meta_client.clone()
     }
+
+    pub fn client_pool(&self) -> ComputeClientPoolRef {
+        self.client_pool.clone()
+    }
 }
diff --git a/src/stream/src/task/mod.rs b/src/stream/src/task/mod.rs
index 77f21b52406f5..b5382b3418052 100644
--- a/src/stream/src/task/mod.rs
+++ b/src/stream/src/task/mod.rs
@@ -19,7 +19,7 @@ use parking_lot::{MappedMutexGuard, Mutex, MutexGuard, RwLock};
 use risingwave_common::config::StreamingConfig;
 use risingwave_common::util::addr::HostAddr;
 use risingwave_pb::common::ActorInfo;
-use risingwave_rpc_client::ComputeClientPool;
+use risingwave_rpc_client::ComputeClientPoolRef;
 
 use crate::error::StreamResult;
 use crate::executor::exchange::permit::{self, Receiver, Sender};
@@ -28,6 +28,7 @@ mod barrier_manager;
 mod env;
 mod stream_manager;
 
+pub(crate) use barrier_manager::SubscribeMutationItem;
 pub use barrier_manager::*;
 pub use env::*;
 pub use stream_manager::*;
@@ -39,6 +40,25 @@ pub type DispatcherId = u64;
 pub type UpDownActorIds = (ActorId, ActorId);
 pub type UpDownFragmentIds = (FragmentId, FragmentId);
 
+#[derive(Hash, Eq, PartialEq, Copy, Clone, Debug)]
+struct PartialGraphId(u32);
+
+impl PartialGraphId {
+    fn new(id: u32) -> Self {
+        Self(id)
+    }
+
+    fn is_global_graph(&self) -> bool {
+        self.0 == u32::MAX
+    }
+}
+
+impl From<PartialGraphId> for u32 {
+    fn from(val: PartialGraphId) -> u32 {
+        val.0
+    }
+}
+
 /// Stores the information which may be modified from the data plane.
 ///
 /// The data structure is created in `LocalBarrierWorker` and is shared by actors created
@@ -75,10 +95,10 @@ pub struct SharedContext {
     /// between two actors/actors.
     pub(crate) addr: HostAddr,
 
-    /// The pool of compute clients.
+    /// Compute client pool for streaming gRPC exchange.
     // TODO: currently the client pool won't be cleared. Should remove compute clients when
     // disconnected.
-    pub(crate) compute_client_pool: ComputeClientPool,
+    pub(crate) compute_client_pool: ComputeClientPoolRef,
 
     pub(crate) config: StreamingConfig,
 
@@ -94,30 +114,28 @@ impl std::fmt::Debug for SharedContext {
 }
 
 impl SharedContext {
-    pub fn new(
-        addr: HostAddr,
-        config: &StreamingConfig,
-        local_barrier_manager: LocalBarrierManager,
-    ) -> Self {
+    pub fn new(env: &StreamEnvironment, local_barrier_manager: LocalBarrierManager) -> Self {
         Self {
             channel_map: Default::default(),
             actor_infos: Default::default(),
-            addr,
-            compute_client_pool: ComputeClientPool::default(),
-            config: config.clone(),
+            addr: env.server_address().clone(),
+            config: env.config().as_ref().to_owned(),
+            compute_client_pool: env.client_pool(),
             local_barrier_manager,
         }
     }
 
     #[cfg(test)]
     pub fn for_test() -> Self {
+        use std::sync::Arc;
+
         use risingwave_common::config::StreamingDeveloperConfig;
+        use risingwave_rpc_client::ComputeClientPool;
 
         Self {
             channel_map: Default::default(),
             actor_infos: Default::default(),
             addr: LOCAL_TEST_ADDR.clone(),
-            compute_client_pool: ComputeClientPool::default(),
             config: StreamingConfig {
                 developer: StreamingDeveloperConfig {
                     exchange_initial_permits: permit::for_test::INITIAL_PERMITS,
@@ -127,6 +145,7 @@ impl SharedContext {
                 },
                 ..Default::default()
             },
+            compute_client_pool: Arc::new(ComputeClientPool::for_test()),
             local_barrier_manager: LocalBarrierManager::for_test(),
         }
     }
diff --git a/src/stream/src/task/stream_manager.rs b/src/stream/src/task/stream_manager.rs
index ce4c82f080561..a8c1c625a5c37 100644
--- a/src/stream/src/task/stream_manager.rs
+++ b/src/stream/src/task/stream_manager.rs
@@ -26,7 +26,7 @@ use futures::stream::BoxStream;
 use futures::FutureExt;
 use itertools::Itertools;
 use risingwave_common::bail;
-use risingwave_common::buffer::Bitmap;
+use risingwave_common::bitmap::Bitmap;
 use risingwave_common::catalog::{Field, Schema, TableId};
 use risingwave_common::config::MetricLevel;
 use risingwave_pb::common::ActorInfo;
@@ -277,6 +277,12 @@ impl LocalStreamManager {
             })
             .await
     }
+
+    pub async fn shutdown(&self) -> StreamResult<()> {
+        self.actor_op_tx
+            .send_and_await(|result_sender| LocalActorOperation::Shutdown { result_sender })
+            .await
+    }
 }
 
 impl LocalBarrierWorker {
diff --git a/src/stream/src/telemetry.rs b/src/stream/src/telemetry.rs
new file mode 100644
index 0000000000000..e64f71fa94e18
--- /dev/null
+++ b/src/stream/src/telemetry.rs
@@ -0,0 +1,37 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use risingwave_common::telemetry::report::report_event_common;
+use risingwave_pb::telemetry::{PbTelemetryDatabaseObject, PbTelemetryEventStage};
+
+const TELEMETRY_COMPUTE_REPORT_TYPE: &str = "compute";
+
+pub fn report_event(
+    event_stage: PbTelemetryEventStage,
+    event_name: &str,
+    catalog_id: i64,
+    connector_name: Option<String>,
+    component: Option<PbTelemetryDatabaseObject>,
+    attributes: Option<jsonbb::Value>, // any json string
+) {
+    report_event_common(
+        event_stage,
+        event_name,
+        catalog_id,
+        connector_name,
+        component,
+        attributes,
+        TELEMETRY_COMPUTE_REPORT_TYPE.to_string(),
+    );
+}
diff --git a/src/stream/tests/README.md b/src/stream/tests/README.md
deleted file mode 120000
index 9b94e2d61ad05..0000000000000
--- a/src/stream/tests/README.md
+++ /dev/null
@@ -1 +0,0 @@
-src/stream/README.md
\ No newline at end of file
diff --git a/src/stream/tests/integration_tests/eowc_over_window.rs b/src/stream/tests/integration_tests/eowc_over_window.rs
index 72e3dade4c993..8365331ad8876 100644
--- a/src/stream/tests/integration_tests/eowc_over_window.rs
+++ b/src/stream/tests/integration_tests/eowc_over_window.rs
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use risingwave_expr::aggregate::{AggArgs, AggKind};
+use risingwave_expr::aggregate::{AggArgs, PbAggKind};
 use risingwave_expr::window_function::{Frame, FrameBound, WindowFuncCall, WindowFuncKind};
 use risingwave_stream::executor::{EowcOverWindowExecutor, EowcOverWindowExecutorArgs};
 
@@ -85,14 +85,14 @@ async fn test_over_window() {
     let calls = vec![
         // lag(x, 1)
         WindowFuncCall {
-            kind: WindowFuncKind::Aggregate(AggKind::FirstValue),
+            kind: WindowFuncKind::Aggregate(PbAggKind::FirstValue.into()),
             args: AggArgs::from_iter([(DataType::Int32, 3)]),
             return_type: DataType::Int32,
             frame: Frame::rows(FrameBound::Preceding(1), FrameBound::Preceding(1)),
         },
         // lead(x, 1)
         WindowFuncCall {
-            kind: WindowFuncKind::Aggregate(AggKind::FirstValue),
+            kind: WindowFuncKind::Aggregate(PbAggKind::FirstValue.into()),
             args: AggArgs::from_iter([(DataType::Int32, 3)]),
             return_type: DataType::Int32,
             frame: Frame::rows(FrameBound::Following(1), FrameBound::Following(1)),
@@ -185,7 +185,7 @@ async fn test_over_window() {
 async fn test_over_window_aggregate() {
     let store = MemoryStateStore::new();
     let calls = vec![WindowFuncCall {
-        kind: WindowFuncKind::Aggregate(AggKind::Sum),
+        kind: WindowFuncKind::Aggregate(PbAggKind::Sum.into()),
         args: AggArgs::from_iter([(DataType::Int32, 3)]),
         return_type: DataType::Int64,
         frame: Frame::rows(FrameBound::Preceding(1), FrameBound::Following(1)),
diff --git a/src/stream/tests/integration_tests/over_window.rs b/src/stream/tests/integration_tests/over_window.rs
index f1ebba752b167..8f96995a8c0d6 100644
--- a/src/stream/tests/integration_tests/over_window.rs
+++ b/src/stream/tests/integration_tests/over_window.rs
@@ -13,7 +13,7 @@
 // limitations under the License.
 
 use risingwave_common::session_config::OverWindowCachePolicy;
-use risingwave_expr::aggregate::{AggArgs, AggKind};
+use risingwave_expr::aggregate::{AggArgs, PbAggKind};
 use risingwave_expr::window_function::{
     Frame, FrameBound, FrameExclusion, WindowFuncCall, WindowFuncKind,
 };
@@ -104,14 +104,14 @@ async fn test_over_window_lag_lead_append_only() {
     let calls = vec![
         // lag(x, 1)
         WindowFuncCall {
-            kind: WindowFuncKind::Aggregate(AggKind::FirstValue),
+            kind: WindowFuncKind::Aggregate(PbAggKind::FirstValue.into()),
             args: AggArgs::from_iter([(DataType::Int32, 3)]),
             return_type: DataType::Int32,
             frame: Frame::rows(FrameBound::Preceding(1), FrameBound::Preceding(1)),
         },
         // lead(x, 1)
         WindowFuncCall {
-            kind: WindowFuncKind::Aggregate(AggKind::FirstValue),
+            kind: WindowFuncKind::Aggregate(PbAggKind::FirstValue.into()),
             args: AggArgs::from_iter([(DataType::Int32, 3)]),
             return_type: DataType::Int32,
             frame: Frame::rows(FrameBound::Following(1), FrameBound::Following(1)),
@@ -215,14 +215,14 @@ async fn test_over_window_lag_lead_with_updates() {
     let calls = vec![
         // lag(x, 1)
         WindowFuncCall {
-            kind: WindowFuncKind::Aggregate(AggKind::FirstValue),
+            kind: WindowFuncKind::Aggregate(PbAggKind::FirstValue.into()),
             args: AggArgs::from_iter([(DataType::Int32, 3)]),
             return_type: DataType::Int32,
             frame: Frame::rows(FrameBound::Preceding(1), FrameBound::Preceding(1)),
         },
         // lead(x, 1)
         WindowFuncCall {
-            kind: WindowFuncKind::Aggregate(AggKind::FirstValue),
+            kind: WindowFuncKind::Aggregate(PbAggKind::FirstValue.into()),
             args: AggArgs::from_iter([(DataType::Int32, 3)]),
             return_type: DataType::Int32,
             frame: Frame::rows(FrameBound::Following(1), FrameBound::Following(1)),
@@ -390,7 +390,7 @@ async fn test_over_window_sum() {
         //   rows between 1 preceding and 2 following exclude current row
         // )
         WindowFuncCall {
-            kind: WindowFuncKind::Aggregate(AggKind::Sum),
+            kind: WindowFuncKind::Aggregate(PbAggKind::Sum.into()),
             args: AggArgs::from_iter([(DataType::Int32, 3)]),
             return_type: DataType::Int64,
             frame: Frame::rows_with_exclusion(
diff --git a/src/tests/compaction_test/src/compaction_test_runner.rs b/src/tests/compaction_test/src/compaction_test_runner.rs
index f294d8fb6fe5a..de5e747624098 100644
--- a/src/tests/compaction_test/src/compaction_test_runner.rs
+++ b/src/tests/compaction_test/src/compaction_test_runner.rs
@@ -30,6 +30,7 @@ use risingwave_common::config::{
 };
 use risingwave_common::util::addr::HostAddr;
 use risingwave_common::util::iter_util::ZipEqFast;
+use risingwave_common::util::tokio_util::sync::CancellationToken;
 use risingwave_hummock_sdk::key::TableKey;
 use risingwave_hummock_sdk::version::{HummockVersion, HummockVersionDelta};
 use risingwave_hummock_sdk::{CompactionGroupId, HummockEpoch, FIRST_VERSION_ID};
@@ -153,7 +154,7 @@ pub async fn start_meta_node(listen_addr: String, state_store: String, config_pa
         "enable_compaction_deterministic should be set"
     );
 
-    risingwave_meta_node::start(meta_opts).await
+    risingwave_meta_node::start(meta_opts, CancellationToken::new() /* dummy */).await
 }
 
 async fn start_compactor_node(
@@ -172,7 +173,7 @@ async fn start_compactor_node(
         "--config-path",
         &config_path,
     ]);
-    risingwave_compactor::start(opts).await
+    risingwave_compactor::start(opts, CancellationToken::new() /* dummy */).await
 }
 
 pub fn start_compactor_thread(
diff --git a/src/tests/compaction_test/src/delete_range_runner.rs b/src/tests/compaction_test/src/delete_range_runner.rs
index 5d6c6ff7e70d1..2d272823d6058 100644
--- a/src/tests/compaction_test/src/delete_range_runner.rs
+++ b/src/tests/compaction_test/src/delete_range_runner.rs
@@ -288,7 +288,7 @@ async fn compaction_test(
         version.id, remote_version.id
     );
     for (group, levels) in &version.levels {
-        let l0 = levels.l0.as_ref().unwrap();
+        let l0 = &levels.l0;
         println!(
             "group-{}: l0 sz: {}, count: {}",
             group,
diff --git a/src/tests/e2e_extended_mode/src/test.rs b/src/tests/e2e_extended_mode/src/test.rs
index 26c9b9d4c2c96..52b0a004096ba 100644
--- a/src/tests/e2e_extended_mode/src/test.rs
+++ b/src/tests/e2e_extended_mode/src/test.rs
@@ -76,6 +76,7 @@ impl TestSuite {
         self.complex_cancel(false).await?;
         self.complex_cancel(true).await?;
         self.subquery_with_param().await?;
+        self.create_mview_with_parameter().await?;
         Ok(())
     }
 
@@ -378,13 +379,36 @@ impl TestSuite {
                 .is_err(),
             true
         );
-        test_eq!(
-            client
-                .query("create materialized view v as select $1", &[])
-                .await
-                .is_err(),
-            true
-        );
+
+        Ok(())
+    }
+
+    async fn create_mview_with_parameter(&self) -> anyhow::Result<()> {
+        let client = self.create_client(false).await?;
+
+        let statement = client
+            .prepare_typed(
+                "create materialized view mv as select $1 as x",
+                &[Type::INT4],
+            )
+            .await?;
+
+        client.execute(&statement, &[&42_i32]).await?;
+
+        let rows = client.query("select * from mv", &[]).await?;
+        test_eq!(rows.len(), 1);
+        test_eq!(rows.first().unwrap().get::<usize, i32>(0), 42);
+
+        // Test renaming mv because it relies on parsing and rewrite the `create MV` query
+        client
+            .execute("alter materialized view mv rename to mv2", &[])
+            .await?;
+
+        let rows = client.query("select * from mv2", &[]).await?;
+        test_eq!(rows.len(), 1);
+        test_eq!(rows.first().unwrap().get::<usize, i32>(0), 42);
+
+        client.execute("drop materialized view mv2", &[]).await?;
 
         Ok(())
     }
diff --git a/src/tests/regress/data/sql/float8.sql b/src/tests/regress/data/sql/float8.sql
index f2e849cba889b..ace482e8495f7 100644
--- a/src/tests/regress/data/sql/float8.sql
+++ b/src/tests/regress/data/sql/float8.sql
@@ -284,12 +284,12 @@ SELECT x,
 FROM (VALUES (0), (45), (90), (135), (180),
       (225), (270), (315), (360)) AS t(x);
 
---@ SELECT x,
---@        asind(x),
---@        asind(x) IN (-90,-30,0,30,90) AS asind_exact,
---@        acosd(x),
---@        acosd(x) IN (0,60,90,120,180) AS acosd_exact
---@ FROM (VALUES (-1), (-0.5), (0), (0.5), (1)) AS t(x);
+SELECT x,
+       asind(x),
+       asind(x) IN (-90,-30,0,30,90) AS asind_exact,
+       acosd(x),
+       acosd(x) IN (0,60,90,120,180) AS acosd_exact
+FROM (VALUES (-1), (-0.5), (0), (0.5), (1)) AS t(x);
 
 --@ SELECT x,
 --@        atand(x),
diff --git a/src/tests/regress/data/sql/strings.sql b/src/tests/regress/data/sql/strings.sql
index af72a06449010..92f36e700b3fc 100644
--- a/src/tests/regress/data/sql/strings.sql
+++ b/src/tests/regress/data/sql/strings.sql
@@ -358,14 +358,14 @@ SELECT 'maca' NOT LIKE 'm%aca' ESCAPE '%' AS "false";
 SELECT 'ma%a' LIKE 'm%a%%a' ESCAPE '%' AS "true";
 SELECT 'ma%a' NOT LIKE 'm%a%%a' ESCAPE '%' AS "false";
 
---@ SELECT 'bear' LIKE 'b_ear' ESCAPE '_' AS "true";
---@ SELECT 'bear' NOT LIKE 'b_ear' ESCAPE '_' AS "false";
---@ 
---@ SELECT 'be_r' LIKE 'b_e__r' ESCAPE '_' AS "true";
---@ SELECT 'be_r' NOT LIKE 'b_e__r' ESCAPE '_' AS "false";
---@ 
---@ SELECT 'be_r' LIKE '__e__r' ESCAPE '_' AS "false";
---@ SELECT 'be_r' NOT LIKE '__e__r' ESCAPE '_' AS "true";
+SELECT 'bear' LIKE 'b_ear' ESCAPE '_' AS "true";
+SELECT 'bear' NOT LIKE 'b_ear' ESCAPE '_' AS "false";
+
+SELECT 'be_r' LIKE 'b_e__r' ESCAPE '_' AS "true";
+SELECT 'be_r' NOT LIKE 'b_e__r' ESCAPE '_' AS "false";
+
+SELECT 'be_r' LIKE '__e__r' ESCAPE '_' AS "false";
+SELECT 'be_r' NOT LIKE '__e__r' ESCAPE '_' AS "true";
 
 
 --
diff --git a/src/tests/regress/src/schedule.rs b/src/tests/regress/src/schedule.rs
index 269a0881ab3d0..0c599ec5a3dfb 100644
--- a/src/tests/regress/src/schedule.rs
+++ b/src/tests/regress/src/schedule.rs
@@ -140,15 +140,15 @@ impl Schedule {
 
         if !different_tests.is_empty() {
             info!(
-                "Risingwave regress tests failed, these tests are different from expected output: {:?}",
+                "RisingWave regress tests failed, these tests are different from expected output: {:?}",
                 different_tests
             );
             bail!(
-                "Risingwave regress tests failed, these tests are different from expected output: {:?}",
+                "RisingWave regress tests failed, these tests are different from expected output: {:?}",
                 different_tests
             )
         } else {
-            info!("Risingwave regress tests passed.");
+            info!("RisingWave regress tests passed.");
             Ok(())
         }
     }
diff --git a/src/tests/simulation/src/cluster.rs b/src/tests/simulation/src/cluster.rs
index e9a3ff6a212f5..6c9db8c48170f 100644
--- a/src/tests/simulation/src/cluster.rs
+++ b/src/tests/simulation/src/cluster.rs
@@ -450,12 +450,19 @@ impl Cluster {
                 "hummock+sim://hummockadmin:hummockadmin@192.168.12.1:9301/hummock001",
                 "--data-directory",
                 "hummock_001",
+                "--temp-secret-file-dir",
+                &format!("./secrets/meta-{i}"),
             ]);
             handle
                 .create_node()
                 .name(format!("meta-{i}"))
                 .ip([192, 168, 1, i as u8].into())
-                .init(move || risingwave_meta_node::start(opts.clone()))
+                .init(move || {
+                    risingwave_meta_node::start(
+                        opts.clone(),
+                        CancellationToken::new(), // dummy
+                    )
+                })
                 .build();
         }
 
@@ -472,6 +479,8 @@ impl Cluster {
                 "0.0.0.0:4566",
                 "--advertise-addr",
                 &format!("192.168.2.{i}:4566"),
+                "--temp-secret-file-dir",
+                &format!("./secrets/frontend-{i}"),
             ]);
             handle
                 .create_node()
@@ -500,6 +509,8 @@ impl Cluster {
                 "6979321856",
                 "--parallelism",
                 &conf.compute_node_cores.to_string(),
+                "--temp-secret-file-dir",
+                &format!("./secrets/compute-{i}"),
             ]);
             handle
                 .create_node()
@@ -530,7 +541,12 @@ impl Cluster {
                 .create_node()
                 .name(format!("compactor-{i}"))
                 .ip([192, 168, 4, i as u8].into())
-                .init(move || risingwave_compactor::start(opts.clone()))
+                .init(move || {
+                    risingwave_compactor::start(
+                        opts.clone(),
+                        CancellationToken::new(), // dummy
+                    )
+                })
                 .build();
         }
 
diff --git a/src/tests/simulation/src/ctl_ext.rs b/src/tests/simulation/src/ctl_ext.rs
index 77ae0ee414e49..9b57673e49c16 100644
--- a/src/tests/simulation/src/ctl_ext.rs
+++ b/src/tests/simulation/src/ctl_ext.rs
@@ -26,12 +26,12 @@ use itertools::Itertools;
 use rand::seq::{IteratorRandom, SliceRandom};
 use rand::{thread_rng, Rng};
 use risingwave_common::catalog::TableId;
-use risingwave_common::hash::ParallelUnitId;
-use risingwave_pb::meta::get_reschedule_plan_request::PbPolicy;
+use risingwave_common::hash::WorkerSlotId;
+use risingwave_hummock_sdk::{CompactionGroupId, HummockSstableId};
 use risingwave_pb::meta::table_fragments::fragment::FragmentDistributionType;
 use risingwave_pb::meta::table_fragments::PbFragment;
 use risingwave_pb::meta::update_worker_node_schedulability_request::Schedulability;
-use risingwave_pb::meta::{GetClusterInfoResponse, GetReschedulePlanResponse};
+use risingwave_pb::meta::GetClusterInfoResponse;
 use risingwave_pb::stream_plan::StreamNode;
 use serde::de::IntoDeserializer;
 
@@ -142,20 +142,53 @@ impl Fragment {
     /// Generate a reschedule plan for the fragment.
     pub fn reschedule(
         &self,
-        remove: impl AsRef<[ParallelUnitId]>,
-        add: impl AsRef<[ParallelUnitId]>,
+        remove: impl AsRef<[WorkerSlotId]>,
+        add: impl AsRef<[WorkerSlotId]>,
     ) -> String {
         let remove = remove.as_ref();
         let add = add.as_ref();
 
-        let mut f = String::new();
-        write!(f, "{}", self.id()).unwrap();
-        if !remove.is_empty() {
-            write!(f, " -{:?}", remove).unwrap();
+        let mut worker_decreased = HashMap::new();
+        for worker_slot in remove {
+            let worker_id = worker_slot.worker_id();
+            *worker_decreased.entry(worker_id).or_insert(0) += 1;
         }
-        if !add.is_empty() {
-            write!(f, " +{:?}", add).unwrap();
+
+        let mut worker_increased = HashMap::new();
+        for worker_slot in add {
+            let worker_id = worker_slot.worker_id();
+            *worker_increased.entry(worker_id).or_insert(0) += 1;
         }
+
+        let worker_ids: HashSet<_> = worker_increased
+            .keys()
+            .chain(worker_decreased.keys())
+            .cloned()
+            .collect();
+
+        let mut worker_actor_diff = HashMap::new();
+
+        for worker_id in worker_ids {
+            let increased = worker_increased.remove(&worker_id).unwrap_or(0);
+            let decreased = worker_decreased.remove(&worker_id).unwrap_or(0);
+            let diff = increased - decreased;
+            if diff != 0 {
+                worker_actor_diff.insert(worker_id, diff);
+            }
+        }
+
+        let mut f = String::new();
+
+        if !worker_actor_diff.is_empty() {
+            let worker_diff_str = worker_actor_diff
+                .into_iter()
+                .map(|(k, v)| format!("{}:{}", k, v))
+                .join(", ");
+
+            write!(f, "{}", self.id()).unwrap();
+            write!(f, ":[{}]", worker_diff_str).unwrap();
+        }
+
         f
     }
 
@@ -163,61 +196,81 @@ impl Fragment {
     ///
     /// Consumes `self` as the actor info will be stale after rescheduling.
     pub fn random_reschedule(self) -> String {
-        let (all_parallel_units, current_parallel_units) = self.parallel_unit_usage();
+        let all_worker_slots = self.all_worker_slots();
+        let used_worker_slots = self.used_worker_slots();
 
         let rng = &mut thread_rng();
-        let target_parallel_unit_count = match self.inner.distribution_type() {
+        let target_worker_slot_count = match self.inner.distribution_type() {
             FragmentDistributionType::Unspecified => unreachable!(),
             FragmentDistributionType::Single => 1,
-            FragmentDistributionType::Hash => rng.gen_range(1..=all_parallel_units.len()),
+            FragmentDistributionType::Hash => rng.gen_range(1..=all_worker_slots.len()),
         };
-        let target_parallel_units: HashSet<_> = all_parallel_units
-            .choose_multiple(rng, target_parallel_unit_count)
-            .copied()
+
+        let target_worker_slots: HashSet<_> = all_worker_slots
+            .into_iter()
+            .choose_multiple(rng, target_worker_slot_count)
+            .into_iter()
             .collect();
 
-        let remove = current_parallel_units
-            .difference(&target_parallel_units)
+        let remove = used_worker_slots
+            .difference(&target_worker_slots)
             .copied()
             .collect_vec();
-        let add = target_parallel_units
-            .difference(&current_parallel_units)
+
+        let add = target_worker_slots
+            .difference(&used_worker_slots)
             .copied()
             .collect_vec();
 
         self.reschedule(remove, add)
     }
 
-    pub fn parallel_unit_usage(&self) -> (Vec<ParallelUnitId>, HashSet<ParallelUnitId>) {
-        let actor_to_parallel_unit: HashMap<_, _> = self
+    pub fn all_worker_count(&self) -> HashMap<u32, usize> {
+        self.r
+            .worker_nodes
+            .iter()
+            .map(|w| (w.id, w.parallelism as usize))
+            .collect()
+    }
+
+    pub fn all_worker_slots(&self) -> HashSet<WorkerSlotId> {
+        self.all_worker_count()
+            .into_iter()
+            .flat_map(|(k, v)| (0..v).map(move |idx| WorkerSlotId::new(k, idx as _)))
+            .collect()
+    }
+
+    pub fn parallelism(&self) -> usize {
+        self.inner.actors.len()
+    }
+
+    pub fn used_worker_count(&self) -> HashMap<u32, usize> {
+        let actor_to_worker: HashMap<_, _> = self
             .r
             .table_fragments
             .iter()
             .flat_map(|tf| {
-                tf.actor_status.iter().map(|(&actor_id, status)| {
-                    (
-                        actor_id,
-                        status.get_parallel_unit().unwrap().id as ParallelUnitId,
-                    )
-                })
+                tf.actor_status
+                    .iter()
+                    .map(|(&actor_id, status)| (actor_id, status.worker_id()))
             })
             .collect();
 
-        let all_parallel_units = self
-            .r
-            .worker_nodes
-            .iter()
-            .flat_map(|n| n.parallel_units.iter())
-            .map(|p| p.id as ParallelUnitId)
-            .collect_vec();
-        let current_parallel_units: HashSet<_> = self
-            .inner
+        self.inner
             .actors
             .iter()
-            .map(|a| actor_to_parallel_unit[&a.actor_id] as ParallelUnitId)
-            .collect();
+            .map(|a| actor_to_worker[&a.actor_id])
+            .fold(HashMap::<u32, usize>::new(), |mut acc, num| {
+                *acc.entry(num).or_insert(0) += 1;
+                acc
+            })
+    }
 
-        (all_parallel_units, current_parallel_units)
+    pub fn used_worker_slots(&self) -> HashSet<WorkerSlotId> {
+        self.used_worker_count()
+            .into_iter()
+            .flat_map(|(k, v)| (0..v).map(move |idx| WorkerSlotId::new(k, idx as _)))
+            .collect()
     }
 }
 
@@ -434,34 +487,47 @@ impl Cluster {
     }
 
     #[cfg_or_panic(madsim)]
-    pub async fn get_reschedule_plan(&self, policy: PbPolicy) -> Result<GetReschedulePlanResponse> {
-        let revision = self
-            .ctl
+    pub async fn split_compaction_group(
+        &mut self,
+        compaction_group_id: CompactionGroupId,
+        table_id: HummockSstableId,
+    ) -> Result<()> {
+        self.ctl
             .spawn(async move {
-                let r = risingwave_ctl::cmd_impl::meta::get_cluster_info(
-                    &risingwave_ctl::common::CtlContext::default(),
-                )
-                .await?;
-
-                Ok::<_, anyhow::Error>(r.revision)
+                let mut command: Vec<String> = vec![
+                    "hummock".into(),
+                    "split-compaction-group".into(),
+                    "--compaction-group-id".into(),
+                    compaction_group_id.to_string(),
+                    "--table-ids".into(),
+                    table_id.to_string(),
+                ];
+                start_ctl(command).await
             })
             .await??;
+        Ok(())
+    }
 
-        let resp = self
-            .ctl
+    #[cfg_or_panic(madsim)]
+    pub async fn trigger_manual_compaction(
+        &mut self,
+        compaction_group_id: CompactionGroupId,
+        level_id: u32,
+    ) -> Result<()> {
+        self.ctl
             .spawn(async move {
-                let r = risingwave_ctl::cmd_impl::meta::get_reschedule_plan(
-                    &risingwave_ctl::common::CtlContext::default(),
-                    policy,
-                    revision,
-                )
-                .await?;
-
-                Ok::<_, anyhow::Error>(r)
+                let mut command: Vec<String> = vec![
+                    "hummock".into(),
+                    "trigger-manual-compaction".into(),
+                    "--compaction-group-id".into(),
+                    compaction_group_id.to_string(),
+                    "--level".into(),
+                    level_id.to_string(),
+                ];
+                start_ctl(command).await
             })
             .await??;
-
-        Ok(resp)
+        Ok(())
     }
 }
 
@@ -473,5 +539,6 @@ where
 {
     let args = std::iter::once("ctl".into()).chain(args.into_iter().map(|s| s.into()));
     let opts = risingwave_ctl::CliOpts::parse_from(args);
-    risingwave_ctl::start_fallible(opts).await
+    let context = risingwave_ctl::common::CtlContext::default();
+    risingwave_ctl::start_fallible(opts, &context).await
 }
diff --git a/src/tests/simulation/src/lib.rs b/src/tests/simulation/src/lib.rs
index ae2614e094a6d..aa6303b8e2f65 100644
--- a/src/tests/simulation/src/lib.rs
+++ b/src/tests/simulation/src/lib.rs
@@ -14,7 +14,6 @@
 
 #![feature(trait_alias)]
 #![feature(lint_reasons)]
-#![feature(lazy_cell)]
 #![feature(let_chains)]
 #![feature(try_blocks)]
 #![feature(register_tool)]
diff --git a/src/tests/simulation/src/main.rs b/src/tests/simulation/src/main.rs
index 31cc488044648..e45c4a3285ac3 100644
--- a/src/tests/simulation/src/main.rs
+++ b/src/tests/simulation/src/main.rs
@@ -13,7 +13,6 @@
 // limitations under the License.
 
 #![cfg_attr(not(madsim), allow(dead_code))]
-#![feature(lazy_cell)]
 
 use std::path::PathBuf;
 
diff --git a/src/tests/simulation/tests/integration_tests/backfill_tests.rs b/src/tests/simulation/tests/integration_tests/backfill_tests.rs
index 1d8f7bb727101..fb7cb3db6fb8b 100644
--- a/src/tests/simulation/tests/integration_tests/backfill_tests.rs
+++ b/src/tests/simulation/tests/integration_tests/backfill_tests.rs
@@ -167,7 +167,7 @@ async fn test_arrangement_backfill_replication() -> Result<()> {
     session
         .run("SET STREAMING_USE_ARRANGEMENT_BACKFILL=true")
         .await?;
-    session.run("SET STREAMING_RATE_LIMIT=30").await?;
+    session.run("SET BACKFILL_RATE_LIMIT=30").await?;
     session
         .run("create materialized view m1 as select * from t")
         .await?;
@@ -251,7 +251,7 @@ async fn test_arrangement_backfill_progress() -> Result<()> {
     // Create arrangement backfill with rate limit
     session.run("SET STREAMING_PARALLELISM=1").await?;
     session.run("SET BACKGROUND_DDL=true").await?;
-    session.run("SET STREAMING_RATE_LIMIT=1").await?;
+    session.run("SET BACKFILL_RATE_LIMIT=1").await?;
     session
         .run("CREATE MATERIALIZED VIEW m1 AS SELECT * FROM t")
         .await?;
@@ -311,7 +311,7 @@ async fn test_enable_arrangement_backfill() -> Result<()> {
 async fn test_recovery_cancels_foreground_ddl() -> Result<()> {
     let mut cluster = Cluster::start(Configuration::enable_arrangement_backfill()).await?;
     let mut session = cluster.start_session();
-    session.run("SET STREAMING_RATE_LIMIT=1").await?;
+    session.run("SET BACKFILL_RATE_LIMIT=1").await?;
     session.run("CREATE TABLE t(v1 int);").await?;
     session
         .run("INSERT INTO t select * from generate_series(1, 100000);")
diff --git a/src/tests/simulation/tests/integration_tests/compaction/mod.rs b/src/tests/simulation/tests/integration_tests/compaction/mod.rs
new file mode 100644
index 0000000000000..0cd8ea767c3ed
--- /dev/null
+++ b/src/tests/simulation/tests/integration_tests/compaction/mod.rs
@@ -0,0 +1,152 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![cfg(madsim)]
+
+use std::time::Duration;
+
+use risingwave_hummock_sdk::CompactionGroupId;
+use risingwave_simulation::cluster::{Cluster, ConfigPath, Configuration, Session};
+
+fn cluster_config(interval_sec: usize) -> Configuration {
+    use std::io::Write;
+    let config_path = {
+        let mut file = tempfile::NamedTempFile::new().expect("failed to create temp config file");
+        file.write_all(
+            format!(
+                "\
+[meta]
+max_heartbeat_interval_secs = 300
+periodic_space_reclaim_compaction_interval_sec = {interval_sec}
+
+[system]
+barrier_interval_ms = 1000
+checkpoint_frequency = 1
+
+[server]
+telemetry_enabled = false
+metrics_level = \"Disabled\"
+        "
+            )
+            .as_bytes(),
+        )
+        .expect("failed to write config file");
+        file.into_temp_path()
+    };
+
+    Configuration {
+        config_path: ConfigPath::Temp(config_path.into()),
+        frontend_nodes: 1,
+        compute_nodes: 1,
+        meta_nodes: 1,
+        compactor_nodes: 4,
+        compute_node_cores: 2,
+        ..Default::default()
+    }
+}
+
+#[tokio::test]
+async fn test_vnode_watermark_reclaim() {
+    // The vnode watermark reclaim will be triggered, so the SST will be reclaimed.
+    let config = crate::compaction::cluster_config(10);
+    let mut cluster = Cluster::start(config).await.unwrap();
+    // wait for the service to be ready
+    tokio::time::sleep(Duration::from_secs(15)).await;
+    let mut session = cluster.start_session();
+
+    let compaction_group_id = test_vnode_watermark_reclaim_impl(&mut cluster, &mut session).await;
+
+    assert_compaction_group_sst_count(compaction_group_id, 6, 1, &mut session).await;
+    // Need wait longer for reclamation, due to the hard-coded STATE_CLEANING_PERIOD_EPOCH.
+    tokio::time::sleep(Duration::from_secs(500)).await;
+    assert_compaction_group_sst_count(compaction_group_id, 6, 0, &mut session).await;
+}
+
+#[tokio::test]
+async fn test_no_vnode_watermark_reclaim() {
+    // The vnode watermark reclaim won't be triggered, so the SST won't be reclaimed.
+    let config = crate::compaction::cluster_config(3600);
+    let mut cluster = Cluster::start(config).await.unwrap();
+    // wait for the service to be ready
+    tokio::time::sleep(Duration::from_secs(15)).await;
+    let mut session = cluster.start_session();
+
+    let compaction_group_id = test_vnode_watermark_reclaim_impl(&mut cluster, &mut session).await;
+
+    assert_compaction_group_sst_count(compaction_group_id, 6, 1, &mut session).await;
+    tokio::time::sleep(Duration::from_secs(500)).await;
+    assert_compaction_group_sst_count(compaction_group_id, 6, 1, &mut session).await;
+}
+
+async fn assert_compaction_group_sst_count(
+    compaction_group_id: CompactionGroupId,
+    level_id: usize,
+    expected: usize,
+    session: &mut Session,
+) {
+    let count = session
+        .run(format!("SELECT COUNT(*) FROM rw_hummock_sstables WHERE compaction_group_id={compaction_group_id} and level_id={level_id};"))
+        .await
+        .unwrap();
+    assert_eq!(count.parse::<usize>().unwrap(), expected);
+}
+
+async fn test_vnode_watermark_reclaim_impl(
+    cluster: &mut Cluster,
+    session: &mut Session,
+) -> CompactionGroupId {
+    session
+        .run("CREATE TABLE t2 (ts timestamptz, v INT);")
+        .await
+        .unwrap();
+    session
+        .run("CREATE MATERIALIZED VIEW mv2 AS SELECT * FROM t2 WHERE ts > now() - INTERVAL '10s';")
+        .await
+        .unwrap();
+
+    let table_id = session
+        .run("SELECT id FROM rw_internal_tables WHERE name LIKE '%dynamicfilterleft%';")
+        .await
+        .unwrap()
+        .parse::<u64>()
+        .unwrap();
+
+    // Move the table to a dedicated group to prevent its vnode watermark from being reclaimed during the compaction of other tables.
+    cluster.split_compaction_group(2, table_id).await.unwrap();
+    tokio::time::sleep(Duration::from_secs(5)).await;
+    let compaction_group_id = session
+        .run("SELECT max(id) FROM rw_hummock_compaction_group_configs;")
+        .await
+        .unwrap()
+        .parse::<u64>()
+        .unwrap();
+    assert!(compaction_group_id > 3);
+
+    session
+        .run("INSERT INTO t2 VALUES (now(), 1);")
+        .await
+        .unwrap();
+    session.run("FLUSH;").await.unwrap();
+    assert_compaction_group_sst_count(compaction_group_id, 0, 1, session).await;
+    assert_compaction_group_sst_count(compaction_group_id, 6, 0, session).await;
+    // Compact data to L6 so that vnode watermark picker can take effects.
+    cluster
+        .trigger_manual_compaction(compaction_group_id, 0)
+        .await
+        .unwrap();
+    tokio::time::sleep(Duration::from_secs(5)).await;
+    assert_compaction_group_sst_count(compaction_group_id, 0, 0, session).await;
+    assert_compaction_group_sst_count(compaction_group_id, 6, 1, session).await;
+    compaction_group_id
+}
diff --git a/src/tests/simulation/tests/integration_tests/main.rs b/src/tests/simulation/tests/integration_tests/main.rs
index 475793a88b709..fe57a2c896f18 100644
--- a/src/tests/simulation/tests/integration_tests/main.rs
+++ b/src/tests/simulation/tests/integration_tests/main.rs
@@ -18,7 +18,6 @@
 //! for the rationale behind this approach.
 
 #![feature(stmt_expr_attributes)]
-#![feature(lazy_cell)]
 #![feature(extract_if)]
 
 mod backfill_tests;
@@ -29,4 +28,5 @@ mod sink;
 mod storage;
 mod throttle;
 
+mod compaction;
 mod utils;
diff --git a/src/tests/simulation/tests/integration_tests/recovery/backfill.rs b/src/tests/simulation/tests/integration_tests/recovery/backfill.rs
index f2e913b3a4903..512841a63c9a2 100644
--- a/src/tests/simulation/tests/integration_tests/recovery/backfill.rs
+++ b/src/tests/simulation/tests/integration_tests/recovery/backfill.rs
@@ -83,13 +83,31 @@ async fn test_snapshot_mv() -> Result<()> {
 
     let id = fragment.id();
 
-    cluster.reschedule(format!("{id}-[1,2,3,4,5]")).await?;
+    let workers = fragment.all_worker_count().into_keys().collect_vec();
+
+    // prev cluster.reschedule(format!("{id}-[1,2,3,4,5]")).await?;
+    cluster
+        .reschedule(format!(
+            "{}:[{}]",
+            id,
+            format_args!("{}:-1,{}:-2,{}:-2", workers[0], workers[1], workers[2]),
+        ))
+        .await?;
+
     sleep(Duration::from_secs(3)).await;
 
     // Before complete recovery should be NO_BACKFILL state
     test_no_backfill_state(&mut session).await?;
 
-    cluster.reschedule(format!("{id}+[1,2,3,4,5]")).await?;
+    // prev cluster.reschedule(format!("{id}+[1,2,3,4,5]")).await?;
+    cluster
+        .reschedule(format!(
+            "{}:[{}]",
+            id,
+            format_args!("{}:1,{}:2,{}:2", workers[0], workers[1], workers[2]),
+        ))
+        .await?;
+
     sleep(Duration::from_secs(3)).await;
 
     // After recovery should be NO_BACKFILL state
@@ -130,7 +148,17 @@ async fn test_backfill_mv() -> Result<()> {
 
     let id = fragment.id();
 
-    cluster.reschedule(format!("{id}-[1,2,3,4,5]")).await?;
+    let workers = fragment.all_worker_count().into_keys().collect_vec();
+
+    // prev cluster.reschedule(format!("{id}-[1,2,3,4,5]")).await?;
+    cluster
+        .reschedule(format!(
+            "{}:[{}]",
+            id,
+            format_args!("{}:-1,{}:-2,{}:-2", workers[0], workers[1], workers[2]),
+        ))
+        .await?;
+
     sleep(Duration::from_secs(3)).await;
 
     let internal_table = session.run(SHOW_INTERNAL_TABLES).await?;
@@ -139,7 +167,15 @@ async fn test_backfill_mv() -> Result<()> {
         .await?;
     assert_eq!(results.lines().collect_vec().len(), 256);
 
-    cluster.reschedule(format!("{id}+[1,2,3,4,5]")).await?;
+    // prev cluster.reschedule(format!("{id}+[1,2,3,4,5]")).await?;
+    cluster
+        .reschedule(format!(
+            "{}:[{}]",
+            id,
+            format_args!("{}:1,{}:2,{}:2", workers[0], workers[1], workers[2]),
+        ))
+        .await?;
+
     sleep(Duration::from_secs(3)).await;
 
     let internal_table = session.run(SHOW_INTERNAL_TABLES).await?;
@@ -183,7 +219,16 @@ async fn test_index_backfill() -> Result<()> {
 
     let id = fragment.id();
 
-    cluster.reschedule(format!("{id}-[1,2,3,4,5]")).await?;
+    let workers = fragment.all_worker_count().into_keys().collect_vec();
+
+    // prev cluster.reschedule(format!("{id}-[1,2,3,4,5]")).await?;
+    cluster
+        .reschedule(format!(
+            "{}:[{}]",
+            id,
+            format_args!("{}:-1,{}:-2,{}:-2", workers[0], workers[1], workers[2]),
+        ))
+        .await?;
     sleep(Duration::from_secs(3)).await;
 
     let internal_table = session.run(SHOW_INTERNAL_TABLES).await?;
@@ -192,7 +237,14 @@ async fn test_index_backfill() -> Result<()> {
         .await?;
     assert_eq!(results.lines().collect_vec().len(), 256);
 
-    cluster.reschedule(format!("{id}+[1,2,3,4,5]")).await?;
+    // prev cluster.reschedule(format!("{id}+[1,2,3,4,5]")).await?;
+    cluster
+        .reschedule(format!(
+            "{}:[{}]",
+            id,
+            format_args!("{}:1,{}:2,{}:2", workers[0], workers[1], workers[2]),
+        ))
+        .await?;
     sleep(Duration::from_secs(3)).await;
 
     let internal_table = session.run(SHOW_INTERNAL_TABLES).await?;
diff --git a/src/tests/simulation/tests/integration_tests/recovery/background_ddl.rs b/src/tests/simulation/tests/integration_tests/recovery/background_ddl.rs
index 80ff76a3f1020..8585867bd75ae 100644
--- a/src/tests/simulation/tests/integration_tests/recovery/background_ddl.rs
+++ b/src/tests/simulation/tests/integration_tests/recovery/background_ddl.rs
@@ -28,9 +28,9 @@ const DROP_TABLE: &str = "DROP TABLE t;";
 const SEED_TABLE_500: &str = "INSERT INTO t SELECT generate_series FROM generate_series(1, 500);";
 const SEED_TABLE_100: &str = "INSERT INTO t SELECT generate_series FROM generate_series(1, 100);";
 const SET_BACKGROUND_DDL: &str = "SET BACKGROUND_DDL=true;";
-const SET_RATE_LIMIT_2: &str = "SET STREAMING_RATE_LIMIT=2;";
-const SET_RATE_LIMIT_1: &str = "SET STREAMING_RATE_LIMIT=1;";
-const RESET_RATE_LIMIT: &str = "SET STREAMING_RATE_LIMIT=DEFAULT;";
+const SET_RATE_LIMIT_2: &str = "SET BACKFILL_RATE_LIMIT=2;";
+const SET_RATE_LIMIT_1: &str = "SET BACKFILL_RATE_LIMIT=1;";
+const RESET_RATE_LIMIT: &str = "SET BACKFILL_RATE_LIMIT=DEFAULT;";
 const CREATE_MV1: &str = "CREATE MATERIALIZED VIEW mv1 as SELECT * FROM t;";
 const DROP_MV1: &str = "DROP MATERIALIZED VIEW mv1;";
 const WAIT: &str = "WAIT;";
@@ -229,7 +229,7 @@ async fn test_ddl_cancel() -> Result<()> {
         let result = create_mv(&mut session).await;
         match result {
             Ok(_) => break,
-            Err(e) if e.to_string().contains("in creating procedure") => {
+            Err(e) if e.to_string().contains("The table is being created") => {
                 tracing::info!("create mv failed, retrying: {}", e);
             }
             Err(e) => {
@@ -262,12 +262,12 @@ async fn test_high_barrier_latency_cancel(config: Configuration) -> Result<()> {
 
     session.run("CREATE TABLE fact1 (v1 int)").await?;
     session
-        .run("INSERT INTO fact1 select 1 from generate_series(1, 100000)")
+        .run("INSERT INTO fact1 select 1 from generate_series(1, 10000)")
         .await?;
 
     session.run("CREATE TABLE fact2 (v1 int)").await?;
     session
-        .run("INSERT INTO fact2 select 1 from generate_series(1, 100000)")
+        .run("INSERT INTO fact2 select 1 from generate_series(1, 10000)")
         .await?;
     session.flush().await?;
 
diff --git a/src/tests/simulation/tests/integration_tests/scale/auto_parallelism.rs b/src/tests/simulation/tests/integration_tests/scale/auto_parallelism.rs
index 982b85d48f4cf..8e2b0f4465509 100644
--- a/src/tests/simulation/tests/integration_tests/scale/auto_parallelism.rs
+++ b/src/tests/simulation/tests/integration_tests/scale/auto_parallelism.rs
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::{HashMap, HashSet};
+use std::collections::HashMap;
 use std::time::Duration;
 
 use anyhow::Result;
@@ -62,37 +62,33 @@ async fn test_passive_online_and_offline() -> Result<()> {
         ])
         .await?;
 
-    let (_, single_used_parallel_unit_ids) = single_agg_fragment.parallel_unit_usage();
+    assert_eq!(single_agg_fragment.parallelism(), 1);
 
-    let used_parallel_unit_id = single_used_parallel_unit_ids.iter().next().unwrap();
+    let used_worker_slots = single_agg_fragment.used_worker_count();
 
-    let mut workers: Vec<WorkerNode> = cluster
+    let (single_used_worker_id, should_be_one) =
+        used_worker_slots.into_iter().exactly_one().unwrap();
+
+    assert_eq!(should_be_one, 1);
+
+    let worker_map: HashMap<_, _> = cluster
         .get_cluster_info()
         .await?
         .worker_nodes
         .into_iter()
         .filter(|worker| worker.r#type() == WorkerType::ComputeNode)
+        .map(|worker| (worker.id, worker))
         .collect();
 
-    let prev_workers = workers
-        .extract_if(|worker| {
-            worker
-                .parallel_units
-                .iter()
-                .map(|parallel_unit| parallel_unit.id)
-                .contains(used_parallel_unit_id)
-        })
-        .collect_vec();
-
-    let prev_worker = prev_workers.into_iter().exactly_one().unwrap();
-    let host = prev_worker.host.unwrap().host;
+    let prev_worker = worker_map.get(&single_used_worker_id).unwrap();
+    let host = prev_worker.clone().host.unwrap().host;
     let host_name = format!("compute-{}", host.split('.').last().unwrap());
 
-    let (all_parallel_units, used_parallel_units) = table_mat_fragment.parallel_unit_usage();
+    let all_worker_slots = table_mat_fragment.all_worker_count();
+    let used_worker_slots = table_mat_fragment.used_worker_count();
+    assert_eq!(all_worker_slots, used_worker_slots);
 
-    assert_eq!(all_parallel_units.len(), used_parallel_units.len());
-
-    let initialized_parallelism = used_parallel_units.len();
+    let initialized_parallelism = table_mat_fragment.parallelism();
 
     assert_eq!(
         initialized_parallelism,
@@ -100,13 +96,11 @@ async fn test_passive_online_and_offline() -> Result<()> {
     );
 
     cluster.simple_kill_nodes(vec![host_name.clone()]).await;
-
     // wait for a while
     sleep(Duration::from_secs(
         MAX_HEARTBEAT_INTERVAL_SECS_CONFIG_FOR_AUTO_SCALE * 2,
     ))
     .await;
-
     let table_mat_fragment = cluster
         .locate_one_fragment(vec![
             identity_contains("materialize"),
@@ -114,22 +108,18 @@ async fn test_passive_online_and_offline() -> Result<()> {
         ])
         .await?;
 
-    let (_, used_parallel_units) = table_mat_fragment.parallel_unit_usage();
-
     assert_eq!(
         initialized_parallelism - config.compute_node_cores,
-        used_parallel_units.len()
+        table_mat_fragment.parallelism()
     );
 
     let stream_scan_fragment = cluster
         .locate_one_fragment(vec![identity_contains("streamTableScan")])
         .await?;
 
-    let (_, used_parallel_units) = stream_scan_fragment.parallel_unit_usage();
-
     assert_eq!(
         initialized_parallelism - config.compute_node_cores,
-        used_parallel_units.len()
+        stream_scan_fragment.parallelism()
     );
 
     let single_agg_fragment = cluster
@@ -139,12 +129,11 @@ async fn test_passive_online_and_offline() -> Result<()> {
         ])
         .await?;
 
-    let (_, used_parallel_units_ids) = single_agg_fragment.parallel_unit_usage();
-
-    assert_eq!(used_parallel_units_ids.len(), 1);
-
-    assert_ne!(single_used_parallel_unit_ids, used_parallel_units_ids);
+    let used_worker_slots = single_agg_fragment.used_worker_count();
 
+    let (curr_used_worker_id, should_be_one) = used_worker_slots.into_iter().exactly_one().unwrap();
+    assert_eq!(should_be_one, 1);
+    assert_ne!(single_used_worker_id, curr_used_worker_id);
     session
         .run("select count(*) from t")
         .await?
@@ -186,17 +175,14 @@ async fn test_passive_online_and_offline() -> Result<()> {
         ])
         .await?;
 
-    let (_, used_parallel_units) = table_mat_fragment.parallel_unit_usage();
-
-    assert_eq!(initialized_parallelism, used_parallel_units.len());
+    assert_eq!(initialized_parallelism, table_mat_fragment.parallelism());
 
     let stream_scan_fragment = cluster
         .locate_one_fragment(vec![identity_contains("streamTableScan")])
         .await?;
 
-    let (_, used_parallel_units) = stream_scan_fragment.parallel_unit_usage();
+    assert_eq!(initialized_parallelism, stream_scan_fragment.parallelism());
 
-    assert_eq!(initialized_parallelism, used_parallel_units.len());
     session
         .run("select count(*) from t")
         .await?
@@ -217,7 +203,6 @@ async fn test_active_online() -> Result<()> {
         true,
     );
     let mut cluster = Cluster::start(config.clone()).await?;
-    let mut session = cluster.start_session();
 
     // Keep one worker reserved for adding later.
     cluster
@@ -229,6 +214,8 @@ async fn test_active_online() -> Result<()> {
     ))
     .await;
 
+    let mut session = cluster.start_session();
+
     session.run("create table t (v1 int);").await?;
     session
         .run("create materialized view m as select count(*) from t;")
@@ -248,14 +235,13 @@ async fn test_active_online() -> Result<()> {
         ])
         .await?;
 
-    let (all_parallel_units, used_parallel_units) = table_mat_fragment.parallel_unit_usage();
+    let all_worker_slots = table_mat_fragment.all_worker_count();
 
-    assert_eq!(all_parallel_units.len(), used_parallel_units.len());
+    let used_worker_slots = table_mat_fragment.used_worker_count();
 
-    assert_eq!(
-        all_parallel_units.len(),
-        (config.compute_nodes - 1) * config.compute_node_cores
-    );
+    assert_eq!(all_worker_slots, used_worker_slots);
+
+    assert_eq!(all_worker_slots.len(), config.compute_nodes - 1);
 
     cluster
         .simple_restart_nodes(vec!["compute-2".to_string()])
@@ -273,14 +259,12 @@ async fn test_active_online() -> Result<()> {
         ])
         .await?;
 
-    let (all_parallel_units, used_parallel_units) = table_mat_fragment.parallel_unit_usage();
+    let all_worker_slots = table_mat_fragment.all_worker_count();
 
-    assert_eq!(all_parallel_units.len(), used_parallel_units.len());
+    let used_worker_slots = table_mat_fragment.used_worker_count();
 
-    assert_eq!(
-        all_parallel_units.len(),
-        config.compute_nodes * config.compute_node_cores
-    );
+    assert_eq!(all_worker_slots, used_worker_slots);
+    assert_eq!(all_worker_slots.len(), config.compute_nodes);
 
     Ok(())
 }
@@ -303,7 +287,6 @@ async fn test_auto_parallelism_control_with_fixed_and_auto_helper(
         enable_auto_parallelism_control,
     );
     let mut cluster = Cluster::start(config.clone()).await?;
-    let mut session = cluster.start_session();
 
     // Keep one worker reserved for adding later.
     let select_worker = "compute-2";
@@ -316,6 +299,8 @@ async fn test_auto_parallelism_control_with_fixed_and_auto_helper(
     ))
     .await;
 
+    let mut session = cluster.start_session();
+
     session.run("create table t (v1 int);").await?;
 
     session
@@ -334,14 +319,11 @@ async fn test_auto_parallelism_control_with_fixed_and_auto_helper(
 
     let table_mat_fragment = locate_table_fragment(&mut cluster).await?;
 
-    let (all_parallel_units, used_parallel_units) = table_mat_fragment.parallel_unit_usage();
-
-    assert_eq!(all_parallel_units.len(), used_parallel_units.len());
+    let all_worker_slots = table_mat_fragment.all_worker_count();
+    let used_worker_slots = table_mat_fragment.used_worker_count();
 
-    assert_eq!(
-        all_parallel_units.len(),
-        (config.compute_nodes - 1) * config.compute_node_cores
-    );
+    assert_eq!(all_worker_slots, used_worker_slots);
+    assert_eq!(all_worker_slots.len(), config.compute_nodes - 1);
 
     session.run("alter table t set parallelism = 3").await?;
 
@@ -352,9 +334,7 @@ async fn test_auto_parallelism_control_with_fixed_and_auto_helper(
 
     let table_mat_fragment = locate_table_fragment(&mut cluster).await?;
 
-    let (_, used_parallel_units) = table_mat_fragment.parallel_unit_usage();
-
-    assert_eq!(used_parallel_units.len(), 3);
+    assert_eq!(table_mat_fragment.parallelism(), 3);
 
     // Keep one worker reserved for adding later.
     cluster
@@ -377,33 +357,18 @@ async fn test_auto_parallelism_control_with_fixed_and_auto_helper(
 
     assert_eq!(workers.len(), 3);
 
-    let parallel_unit_to_worker = workers
-        .into_iter()
-        .flat_map(|worker| {
-            worker
-                .parallel_units
-                .into_iter()
-                .map(move |parallel_unit| (parallel_unit.id, worker.id))
-        })
-        .collect::<HashMap<_, _>>();
-
     let table_mat_fragment = locate_table_fragment(&mut cluster).await?;
 
-    let (_, used_parallel_units) = table_mat_fragment.parallel_unit_usage();
+    let used_worker_slots = table_mat_fragment.used_worker_count();
 
-    assert_eq!(used_parallel_units.len(), 3);
-
-    let worker_ids: HashSet<_> = used_parallel_units
-        .iter()
-        .map(|id| parallel_unit_to_worker.get(id).unwrap())
-        .collect();
+    assert_eq!(table_mat_fragment.parallelism(), 3);
 
     // check auto scale out for fixed
     if enable_auto_parallelism_control {
-        assert_eq!(worker_ids.len(), config.compute_nodes);
+        assert_eq!(used_worker_slots.len(), config.compute_nodes);
     } else {
         // no rebalance process
-        assert_eq!(worker_ids.len(), config.compute_nodes - 1);
+        assert_eq!(used_worker_slots.len(), config.compute_nodes - 1);
     }
 
     // We kill compute-2 again to verify the behavior of auto scale-in
@@ -418,16 +383,11 @@ async fn test_auto_parallelism_control_with_fixed_and_auto_helper(
 
     let table_mat_fragment = locate_table_fragment(&mut cluster).await?;
 
-    let (_, used_parallel_units) = table_mat_fragment.parallel_unit_usage();
+    let used_worker_slots = table_mat_fragment.used_worker_count();
 
-    assert_eq!(used_parallel_units.len(), 3);
+    assert_eq!(table_mat_fragment.parallelism(), 3);
 
-    let worker_ids: HashSet<_> = used_parallel_units
-        .iter()
-        .map(|id| parallel_unit_to_worker.get(id).unwrap())
-        .collect();
-
-    assert_eq!(worker_ids.len(), config.compute_nodes - 1);
+    assert_eq!(used_worker_slots.len(), config.compute_nodes - 1);
 
     // We alter parallelism back to auto
 
@@ -442,14 +402,11 @@ async fn test_auto_parallelism_control_with_fixed_and_auto_helper(
 
     let table_mat_fragment = locate_table_fragment(&mut cluster).await?;
 
-    let (all_parallel_units, used_parallel_units) = table_mat_fragment.parallel_unit_usage();
-
-    assert_eq!(all_parallel_units.len(), used_parallel_units.len());
+    let all_worker_slots = table_mat_fragment.all_worker_count();
+    let used_worker_slots = table_mat_fragment.used_worker_count();
 
-    assert_eq!(
-        all_parallel_units.len(),
-        (config.compute_nodes - 1) * config.compute_node_cores
-    );
+    assert_eq!(all_worker_slots, used_worker_slots);
+    assert_eq!(all_worker_slots.len(), config.compute_nodes - 1);
 
     // Keep one worker reserved for adding later.
     cluster
@@ -463,21 +420,15 @@ async fn test_auto_parallelism_control_with_fixed_and_auto_helper(
 
     let table_mat_fragment = locate_table_fragment(&mut cluster).await?;
 
-    let (all_parallel_units, used_parallel_units) = table_mat_fragment.parallel_unit_usage();
+    let all_worker_slots = table_mat_fragment.all_worker_count();
+    let used_worker_slots = table_mat_fragment.used_worker_count();
 
     // check auto scale out for auto
     if enable_auto_parallelism_control {
-        assert_eq!(all_parallel_units.len(), used_parallel_units.len());
-
-        assert_eq!(
-            all_parallel_units.len(),
-            config.compute_nodes * config.compute_node_cores
-        );
+        assert_eq!(all_worker_slots, used_worker_slots);
+        assert_eq!(all_worker_slots.len(), config.compute_nodes);
     } else {
-        assert_eq!(
-            used_parallel_units.len(),
-            (config.compute_nodes - 1) * config.compute_node_cores
-        );
+        assert_eq!(used_worker_slots.len(), config.compute_nodes - 1);
     }
 
     Ok(())
@@ -490,10 +441,6 @@ async fn test_compatibility_with_low_level() -> Result<()> {
         true,
     );
     let mut cluster = Cluster::start(config.clone()).await?;
-    let mut session = cluster.start_session();
-    session
-        .run("SET streaming_use_arrangement_backfill = false;")
-        .await?;
 
     // Keep one worker reserved for adding later.
     let select_worker = "compute-2";
@@ -506,6 +453,11 @@ async fn test_compatibility_with_low_level() -> Result<()> {
     ))
     .await;
 
+    let mut session = cluster.start_session();
+    session
+        .run("SET streaming_use_arrangement_backfill = false;")
+        .await?;
+
     session.run("create table t(v int);").await?;
 
     // single fragment downstream
@@ -530,18 +482,19 @@ async fn test_compatibility_with_low_level() -> Result<()> {
         ])
         .await?;
 
-    let (mut all_parallel_units, _) = table_mat_fragment.parallel_unit_usage();
+    let mut all_workers = table_mat_fragment
+        .all_worker_count()
+        .into_keys()
+        .collect_vec();
 
-    let chosen_parallel_unit_a = all_parallel_units.pop().unwrap();
-    let chosen_parallel_unit_b = all_parallel_units.pop().unwrap();
+    let chosen_worker_a = all_workers.pop().unwrap();
+    let chosen_worker_b = all_workers.pop().unwrap();
 
     let table_mat_fragment_id = table_mat_fragment.id();
 
     // manual scale in table materialize fragment
     cluster
-        .reschedule(format!(
-            "{table_mat_fragment_id}-[{chosen_parallel_unit_a}]",
-        ))
+        .reschedule(format!("{table_mat_fragment_id}:[{chosen_worker_a}:-1]",))
         .await?;
 
     session
@@ -565,9 +518,7 @@ async fn test_compatibility_with_low_level() -> Result<()> {
 
     // manual scale in m_simple materialize fragment
     cluster
-        .reschedule_resolve_no_shuffle(format!(
-            "{simple_mv_fragment_id}-[{chosen_parallel_unit_b}]",
-        ))
+        .reschedule_resolve_no_shuffle(format!("{simple_mv_fragment_id}:[{chosen_worker_b}:-1]",))
         .await?;
 
     // Since `m_simple` only has 1 fragment, and this fragment is a downstream of NO_SHUFFLE relation,
@@ -592,9 +543,7 @@ async fn test_compatibility_with_low_level() -> Result<()> {
 
     // manual scale in m_join materialize fragment
     cluster
-        .reschedule_resolve_no_shuffle(format!(
-            "{hash_join_fragment_id}-[{chosen_parallel_unit_a}]"
-        ))
+        .reschedule_resolve_no_shuffle(format!("{hash_join_fragment_id}:[{chosen_worker_a}:-1]"))
         .await?;
 
     session
@@ -631,7 +580,6 @@ async fn test_compatibility_with_low_level_and_arrangement_backfill() -> Result<
         true,
     );
     let mut cluster = Cluster::start(config.clone()).await?;
-    let mut session = cluster.start_session();
 
     // Keep one worker reserved for adding later.
     let select_worker = "compute-2";
@@ -644,6 +592,8 @@ async fn test_compatibility_with_low_level_and_arrangement_backfill() -> Result<
     ))
     .await;
 
+    let mut session = cluster.start_session();
+
     session.run("create table t(v int);").await?;
 
     // Streaming arrangement backfill
@@ -667,18 +617,19 @@ async fn test_compatibility_with_low_level_and_arrangement_backfill() -> Result<
         ])
         .await?;
 
-    let (mut all_parallel_units, _) = table_mat_fragment.parallel_unit_usage();
+    let mut all_workers = table_mat_fragment
+        .all_worker_count()
+        .into_keys()
+        .collect_vec();
 
-    let chosen_parallel_unit_a = all_parallel_units.pop().unwrap();
-    let chosen_parallel_unit_b = all_parallel_units.pop().unwrap();
+    let chosen_worker_a = all_workers.pop().unwrap();
+    let chosen_worker_b = all_workers.pop().unwrap();
 
     let table_mat_fragment_id = table_mat_fragment.id();
 
     // manual scale in table materialize fragment
     cluster
-        .reschedule(format!(
-            "{table_mat_fragment_id}-[{chosen_parallel_unit_a}]",
-        ))
+        .reschedule(format!("{table_mat_fragment_id}:[{chosen_worker_a}:-1]",))
         .await?;
 
     session
@@ -704,9 +655,7 @@ async fn test_compatibility_with_low_level_and_arrangement_backfill() -> Result<
 
     // manual scale in m_simple materialize fragment
     cluster
-        .reschedule_resolve_no_shuffle(format!(
-            "{simple_mv_fragment_id}-[{chosen_parallel_unit_b}]",
-        ))
+        .reschedule_resolve_no_shuffle(format!("{simple_mv_fragment_id}:[{chosen_worker_b}:-1]",))
         .await?;
 
     // The downstream table fragment should be separate from the upstream table fragment.
diff --git a/src/tests/simulation/tests/integration_tests/scale/cascade_materialized_view.rs b/src/tests/simulation/tests/integration_tests/scale/cascade_materialized_view.rs
index 981f79103403d..f5fdff15addc1 100644
--- a/src/tests/simulation/tests/integration_tests/scale/cascade_materialized_view.rs
+++ b/src/tests/simulation/tests/integration_tests/scale/cascade_materialized_view.rs
@@ -48,11 +48,18 @@ async fn test_simple_cascade_materialized_view() -> Result<()> {
 
     let id = fragment.id();
 
-    cluster.reschedule(format!("{id}-[1,2,3,4,5]")).await?;
+    let all_workers = fragment.all_worker_count().into_keys().collect_vec();
+
+    cluster
+        .reschedule(format!(
+            "{id}:[{}]",
+            all_workers.iter().map(|w| format!("{w}:-1")).join(",")
+        ))
+        .await?;
     sleep(Duration::from_secs(3)).await;
 
     let fragment = cluster.locate_fragment_by_id(id).await?;
-    assert_eq!(fragment.inner.actors.len(), 1);
+    assert_eq!(fragment.inner.actors.len(), 3);
 
     let chain_fragment = cluster
         .locate_one_fragment([identity_contains("StreamTableScan")])
@@ -60,9 +67,9 @@ async fn test_simple_cascade_materialized_view() -> Result<()> {
 
     if arrangement_backfill_is_enabled {
         // The chain fragment is in a different table fragment.
-        assert_eq!(chain_fragment.inner.actors.len(), 6,);
+        assert_eq!(chain_fragment.inner.actors.len(), 6);
         // The upstream materialized fragment should be scaled in
-        assert_eq!(fragment.inner.actors.len(), 1,);
+        assert_eq!(fragment.inner.actors.len(), 3);
     } else {
         // No shuffle, so the fragment of upstream materialized node is the same
         // as stream table scan.
@@ -87,7 +94,12 @@ async fn test_simple_cascade_materialized_view() -> Result<()> {
         .await?
         .assert_result_eq("5");
 
-    cluster.reschedule(format!("{id}+[1,2,3,4,5]")).await?;
+    cluster
+        .reschedule(format!(
+            "{id}:[{}]",
+            all_workers.iter().map(|w| format!("{w}:1")).join(",")
+        ))
+        .await?;
     sleep(Duration::from_secs(3)).await;
 
     let fragment = cluster.locate_fragment_by_id(id).await?;
@@ -147,11 +159,19 @@ async fn test_diamond_cascade_materialized_view() -> Result<()> {
 
     let id = fragment.id();
 
-    cluster.reschedule(format!("{id}-[1,2,3,4,5]")).await?;
+    let all_workers = fragment.all_worker_count().into_keys().collect_vec();
+
+    cluster
+        .reschedule(format!(
+            "{id}:[{}]",
+            all_workers.iter().map(|w| format!("{w}:-1")).join(",")
+        ))
+        .await?;
+
     sleep(Duration::from_secs(3)).await;
 
     let fragment = cluster.locate_fragment_by_id(id).await?;
-    assert_eq!(fragment.inner.actors.len(), 1);
+    assert_eq!(fragment.inner.actors.len(), 3);
 
     session
         .run(&format!(
@@ -166,7 +186,13 @@ async fn test_diamond_cascade_materialized_view() -> Result<()> {
         .await?
         .assert_result_eq("0");
 
-    cluster.reschedule(format!("{id}+[1,2,3,4,5]")).await?;
+    cluster
+        .reschedule(format!(
+            "{id}:[{}]",
+            all_workers.iter().map(|w| format!("{w}:1")).join(",")
+        ))
+        .await?;
+
     sleep(Duration::from_secs(3)).await;
 
     let fragment = cluster.locate_fragment_by_id(id).await?;
diff --git a/src/tests/simulation/tests/integration_tests/scale/dynamic_filter.rs b/src/tests/simulation/tests/integration_tests/scale/dynamic_filter.rs
index a6853b3515c1a..87a0f843682c0 100644
--- a/src/tests/simulation/tests/integration_tests/scale/dynamic_filter.rs
+++ b/src/tests/simulation/tests/integration_tests/scale/dynamic_filter.rs
@@ -16,6 +16,7 @@ use std::collections::HashSet;
 use std::time::Duration;
 
 use anyhow::Result;
+use itertools::Itertools;
 use risingwave_simulation::cluster::{Cluster, Configuration};
 use risingwave_simulation::ctl_ext::predicate::identity_contains;
 use risingwave_simulation::utils::AssertResult;
@@ -56,7 +57,16 @@ async fn test_dynamic_filter() -> Result<()> {
 
     let id = fragment.id();
 
-    cluster.reschedule(format!("{id}-[1,2,3]")).await?;
+    let (worker_1, worker_2, worker_3) = fragment
+        .all_worker_count()
+        .into_keys()
+        .collect_tuple::<(_, _, _)>()
+        .unwrap();
+
+    // prev -[1,2,3]
+    cluster
+        .reschedule(format!("{id}:[{worker_1}:-2, {worker_2}:-1]"))
+        .await?;
     sleep(Duration::from_secs(3)).await;
 
     session.run(SELECT).await?.assert_result_eq("");
@@ -68,7 +78,10 @@ async fn test_dynamic_filter() -> Result<()> {
     // 2
     // 3
 
-    cluster.reschedule(format!("{id}-[4,5]+[1,2,3]")).await?;
+    // prev -[4,5]+[1,2,3]
+    cluster
+        .reschedule(format!("{id}:[{worker_3}:-1, {worker_1}:2]"))
+        .await?;
     sleep(Duration::from_secs(3)).await;
     session.run(SELECT).await?.assert_result_eq("1\n2\n3");
 
@@ -78,7 +91,10 @@ async fn test_dynamic_filter() -> Result<()> {
     session.run(SELECT).await?.assert_result_eq("3");
     // 3
 
-    cluster.reschedule(format!("{id}-[1,2,3]+[4,5]")).await?;
+    // prev -[1,2,3]+[4,5]
+    cluster
+        .reschedule(format!("{id}:[{worker_1}:-2, {worker_3}:1]"))
+        .await?;
     sleep(Duration::from_secs(3)).await;
     session.run(SELECT).await?.assert_result_eq("3");
 
@@ -89,7 +105,10 @@ async fn test_dynamic_filter() -> Result<()> {
     // 2
     // 3
     //
-    cluster.reschedule(format!("{id}+[1,2,3]")).await?;
+    // prev +[1,2,3]
+    cluster
+        .reschedule(format!("{id}:[{worker_1}:2, {worker_2}:1]"))
+        .await?;
     sleep(Duration::from_secs(3)).await;
     session.run(SELECT).await?.assert_result_eq("2\n3");
 
@@ -98,7 +117,8 @@ async fn test_dynamic_filter() -> Result<()> {
     sleep(Duration::from_secs(5)).await;
     session.run(SELECT).await?.assert_result_eq("");
 
-    cluster.reschedule(format!("{id}-[1]")).await?;
+    // prev -[1]
+    cluster.reschedule(format!("{id}:[{worker_1}:1]")).await?;
     sleep(Duration::from_secs(3)).await;
     session.run(SELECT).await?.assert_result_eq("");
 
diff --git a/src/tests/simulation/tests/integration_tests/scale/mod.rs b/src/tests/simulation/tests/integration_tests/scale/mod.rs
index ced22e6875465..f6940f072409e 100644
--- a/src/tests/simulation/tests/integration_tests/scale/mod.rs
+++ b/src/tests/simulation/tests/integration_tests/scale/mod.rs
@@ -19,7 +19,6 @@ mod nexmark_chaos;
 mod nexmark_q4;
 mod nexmark_source;
 mod no_shuffle;
-mod plan;
 mod schedulability;
 mod singleton_migration;
 mod sink;
diff --git a/src/tests/simulation/tests/integration_tests/scale/nexmark_q4.rs b/src/tests/simulation/tests/integration_tests/scale/nexmark_q4.rs
index 6408a8d47f951..e2b84d5a62842 100644
--- a/src/tests/simulation/tests/integration_tests/scale/nexmark_q4.rs
+++ b/src/tests/simulation/tests/integration_tests/scale/nexmark_q4.rs
@@ -15,6 +15,8 @@
 use std::time::Duration;
 
 use anyhow::Result;
+use itertools::Itertools;
+use risingwave_common::hash::WorkerSlotId;
 use risingwave_simulation::cluster::Configuration;
 use risingwave_simulation::ctl_ext::predicate::{
     identity_contains, upstream_fragment_count, BoxedPredicate,
@@ -69,7 +71,7 @@ async fn nexmark_q4_common(predicates: impl IntoIterator<Item = BoxedPredicate>)
     let mut cluster = init().await?;
 
     let fragment = cluster.locate_one_fragment(predicates).await?;
-    let id = fragment.id();
+    let workers = fragment.all_worker_count().into_keys().collect_vec();
 
     // 0s
     wait_initial_data(&mut cluster)
@@ -77,13 +79,32 @@ async fn nexmark_q4_common(predicates: impl IntoIterator<Item = BoxedPredicate>)
         .assert_result_ne(RESULT);
 
     // 0~10s
-    cluster.reschedule(format!("{id}-[0,1]")).await?;
+    cluster
+        .reschedule(fragment.reschedule(
+            [
+                WorkerSlotId::new(workers[0], 0),
+                WorkerSlotId::new(workers[0], 1),
+            ],
+            [],
+        ))
+        .await?;
 
     sleep(Duration::from_secs(5)).await;
 
     // 5~15s
     cluster.run(SELECT).await?.assert_result_ne(RESULT);
-    cluster.reschedule(format!("{id}-[2,3]+[0,1]")).await?;
+    cluster
+        .reschedule(fragment.reschedule(
+            [
+                WorkerSlotId::new(workers[1], 0),
+                WorkerSlotId::new(workers[1], 1),
+            ],
+            [
+                WorkerSlotId::new(workers[0], 0),
+                WorkerSlotId::new(workers[0], 1),
+            ],
+        ))
+        .await?;
 
     sleep(Duration::from_secs(20)).await;
 
@@ -138,6 +159,9 @@ async fn nexmark_q4_cascade() -> Result<()> {
         .await?;
     let id_2 = fragment_2.id();
 
+    // todo, fragment_1's worker
+    let workers = fragment_1.all_worker_count().into_keys().collect_vec();
+
     // 0s
     wait_initial_data(&mut cluster)
         .await?
@@ -145,7 +169,13 @@ async fn nexmark_q4_cascade() -> Result<()> {
 
     // 0~10s
     cluster
-        .reschedule(format!("{id_1}-[0,1]; {id_2}-[0,2,4]"))
+        .reschedule(format!(
+            "{}:[{}];{}:[{}]",
+            fragment_1.id(),
+            format_args!("{}:-2", workers[0]),
+            fragment_2.id(),
+            format_args!("{}:-1,{}:-1,{}:-1", workers[0], workers[1], workers[2]),
+        ))
         .await?;
 
     sleep(Duration::from_secs(5)).await;
@@ -153,7 +183,15 @@ async fn nexmark_q4_cascade() -> Result<()> {
     // 5~15s
     cluster.run(SELECT).await?.assert_result_ne(RESULT);
     cluster
-        .reschedule(format!("{id_1}-[2,4]+[0,1]; {id_2}-[3]+[0,4]"))
+        .reschedule(format!(
+            "{}:[{},{}];{}:[{},{}]",
+            id_1,
+            format_args!("{}:-1,{}:-1", workers[1], workers[2]),
+            format_args!("{}:2", workers[0]),
+            id_2,
+            format_args!("{}:-1", workers[1]),
+            format_args!("{}:1,{}:1", workers[0], workers[2]),
+        ))
         .await?;
 
     sleep(Duration::from_secs(20)).await;
@@ -176,14 +214,27 @@ async fn nexmark_q4_materialize_agg_cache_invalidation() -> Result<()> {
         ])
         .await?;
     let id = fragment.id();
+    let workers = fragment.all_worker_count().into_keys().collect_vec();
 
-    // Let parallel unit 0 handle all groups.
-    cluster.reschedule(format!("{id}-[1,2,3,4,5]")).await?;
+    // Let worker slot 0 handle all groups.
+    cluster
+        .reschedule(format!(
+            "{}:[{}]",
+            id,
+            format_args!("{}:-1,{}:-2,{}:-2", workers[0], workers[1], workers[2]),
+        ))
+        .await?;
     sleep(Duration::from_secs(7)).await;
     let result_1 = cluster.run(SELECT).await?.assert_result_ne(RESULT);
 
     // Scale out.
-    cluster.reschedule(format!("{id}+[1,2,3,4,5]")).await?;
+    cluster
+        .reschedule(format!(
+            "{}:[{}]",
+            id,
+            format_args!("{}:1,{}:2,{}:2", workers[0], workers[1], workers[2]),
+        ))
+        .await?;
     sleep(Duration::from_secs(7)).await;
     cluster
         .run(SELECT)
@@ -191,10 +242,16 @@ async fn nexmark_q4_materialize_agg_cache_invalidation() -> Result<()> {
         .assert_result_ne(result_1)
         .assert_result_ne(RESULT);
 
-    // Let parallel unit 0 handle all groups again.
-    // Note that there're only 5 groups, so if the parallel unit 0 doesn't invalidate the cache
+    // Let worker slot 0 handle all groups again.
+    // Note that there're only 5 groups, so if the worker slot 0 doesn't invalidate the cache
     // correctly, it will yield the wrong result.
-    cluster.reschedule(format!("{id}-[1,2,3,4,5]")).await?;
+    cluster
+        .reschedule(format!(
+            "{}:[{}]",
+            id,
+            format_args!("{}:-1,{}:-2,{}:-2", workers[0], workers[1], workers[2]),
+        ))
+        .await?;
     sleep(Duration::from_secs(20)).await;
 
     cluster.run(SELECT).await?.assert_result_eq(RESULT);
diff --git a/src/tests/simulation/tests/integration_tests/scale/no_shuffle.rs b/src/tests/simulation/tests/integration_tests/scale/no_shuffle.rs
index a409443371ba9..e87659d4f54db 100644
--- a/src/tests/simulation/tests/integration_tests/scale/no_shuffle.rs
+++ b/src/tests/simulation/tests/integration_tests/scale/no_shuffle.rs
@@ -14,6 +14,7 @@
 
 use anyhow::Result;
 use itertools::Itertools;
+use risingwave_common::hash::WorkerSlotId;
 use risingwave_simulation::cluster::{Cluster, Configuration};
 use risingwave_simulation::ctl_ext::predicate::{identity_contains, no_identity_contains};
 use risingwave_simulation::utils::AssertResult;
@@ -80,40 +81,66 @@ async fn test_delta_join() -> Result<()> {
                 .assert_result_eq(result);
 
             #[allow(unused_assignments)]
-            test_times += 1;
+            {
+                test_times += 1;
+            }
         };
     }
 
     test_works!();
 
+    let workers = union_fragment.all_worker_count().into_keys().collect_vec();
     // Scale-in one side
-    cluster.reschedule(format!("{}-[0]", t1.id())).await?;
+    cluster
+        .reschedule(format!("{}:[{}:-1]", t1.id(), workers[0]))
+        .await?;
+
     test_works!();
 
     // Scale-in both sides together
     cluster
-        .reschedule(format!("{}-[2];{}-[0,2]", t1.id(), t2.id()))
+        .reschedule(format!(
+            "{}:[{}];{}:[{}]",
+            t1.id(),
+            format_args!("{}:-1", workers[1]),
+            t2.id(),
+            format_args!("{}:-1, {}:-1", workers[0], workers[1])
+        ))
         .await?;
     test_works!();
 
     // Scale-out one side
-    cluster.reschedule(format!("{}+[0]", t2.id())).await?;
+    cluster
+        .reschedule(format!("{}:[{}:1]", t2.id(), workers[0]))
+        .await?;
     test_works!();
 
     // Scale-out both sides together
     cluster
-        .reschedule(format!("{}+[0,2];{}+[2]", t1.id(), t2.id()))
+        .reschedule(format!(
+            "{}:[{}];{}:[{}]",
+            t1.id(),
+            format_args!("{}:1,{}:1", workers[0], workers[1]),
+            t2.id(),
+            format_args!("{}:1", workers[1]),
+        ))
         .await?;
     test_works!();
 
     // Scale-in join with union
     cluster
-        .reschedule(format!("{}-[5];{}-[5]", t1.id(), union_fragment.id()))
+        .reschedule(format!(
+            "{}:[{}];{}:[{}]",
+            t1.id(),
+            format_args!("{}:-1", workers[2]),
+            t2.id(),
+            format_args!("{}:-1", workers[2])
+        ))
         .await?;
     test_works!();
 
     let result = cluster
-        .reschedule(format!("{}-[0]", lookup_fragments[0].id()))
+        .reschedule(format!("{}:[{}:-1]", lookup_fragments[0].id(), workers[0]))
         .await;
     assert!(
         result.is_err(),
@@ -137,8 +164,15 @@ async fn test_share_multiple_no_shuffle_upstream() -> Result<()> {
         .locate_one_fragment([identity_contains("hashagg")])
         .await?;
 
-    cluster.reschedule(fragment.reschedule([0], [])).await?;
-    cluster.reschedule(fragment.reschedule([], [0])).await?;
+    let workers = fragment.all_worker_count().into_keys().collect_vec();
+
+    cluster
+        .reschedule(fragment.reschedule([WorkerSlotId::new(workers[0], 0)], []))
+        .await?;
+
+    cluster
+        .reschedule(fragment.reschedule([], [WorkerSlotId::new(workers[0], 0)]))
+        .await?;
 
     Ok(())
 }
@@ -157,15 +191,20 @@ async fn test_resolve_no_shuffle_upstream() -> Result<()> {
         .locate_one_fragment([identity_contains("StreamTableScan")])
         .await?;
 
-    let result = cluster.reschedule(fragment.reschedule([0], [])).await;
+    let workers = fragment.all_worker_count().into_keys().collect_vec();
+
+    let result = cluster
+        .reschedule(fragment.reschedule([WorkerSlotId::new(workers[0], 0)], []))
+        .await;
 
     assert!(result.is_err());
 
     cluster
-        .reschedule_resolve_no_shuffle(fragment.reschedule([0], []))
+        .reschedule_resolve_no_shuffle(fragment.reschedule([WorkerSlotId::new(workers[0], 0)], []))
         .await?;
+
     cluster
-        .reschedule_resolve_no_shuffle(fragment.reschedule([], [0]))
+        .reschedule_resolve_no_shuffle(fragment.reschedule([], [WorkerSlotId::new(workers[0], 0)]))
         .await?;
 
     Ok(())
diff --git a/src/tests/simulation/tests/integration_tests/scale/plan.rs b/src/tests/simulation/tests/integration_tests/scale/plan.rs
deleted file mode 100644
index d39e159fc61d9..0000000000000
--- a/src/tests/simulation/tests/integration_tests/scale/plan.rs
+++ /dev/null
@@ -1,332 +0,0 @@
-// Copyright 2024 RisingWave Labs
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::collections::HashMap;
-
-use anyhow::Result;
-use itertools::Itertools;
-use rand::seq::SliceRandom;
-use risingwave_pb::common::{WorkerNode, WorkerType};
-use risingwave_pb::meta::get_reschedule_plan_request::Policy::StableResizePolicy;
-use risingwave_pb::meta::get_reschedule_plan_request::{
-    PbPolicy, PbStableResizePolicy, WorkerChanges,
-};
-use risingwave_pb::meta::PbReschedule;
-use risingwave_simulation::cluster::{Cluster, Configuration};
-use risingwave_simulation::ctl_ext::predicate::{identity_contains, no_identity_contains};
-
-#[tokio::test]
-async fn test_resize_normal() -> Result<()> {
-    let mut cluster = Cluster::start(Configuration::for_scale()).await?;
-    let mut session = cluster.start_session();
-
-    session.run("create table t1 (v1 int);").await?;
-    session.run("create table t2 (v2 int);").await?;
-    session
-        .run("create materialized view mv as select * from t1 join t2 on t1.v1 = t2.v2;")
-        .await?;
-
-    let join_fragment = cluster
-        .locate_one_fragment([identity_contains("hashJoin")])
-        .await?;
-
-    let join_fragment_id = join_fragment.inner.fragment_id;
-
-    let mut workers: Vec<WorkerNode> = cluster
-        .get_cluster_info()
-        .await?
-        .worker_nodes
-        .into_iter()
-        .filter(|worker| worker.r#type() == WorkerType::ComputeNode)
-        .collect();
-
-    workers.pop();
-
-    let removed_workers = workers.iter().map(|worker| worker.id).collect_vec();
-
-    let resp = cluster
-        .get_reschedule_plan(PbPolicy::StableResizePolicy(PbStableResizePolicy {
-            fragment_worker_changes: HashMap::from([(
-                join_fragment_id,
-                WorkerChanges {
-                    include_worker_ids: vec![],
-                    exclude_worker_ids: removed_workers,
-                    ..Default::default()
-                },
-            )]),
-        }))
-        .await?;
-
-    let reschedules = resp.reschedules;
-    assert_eq!(reschedules.len(), 1);
-    let target_plan: PbReschedule = reschedules.get(&join_fragment_id).unwrap().clone();
-
-    assert_eq!(target_plan.added_parallel_units.len(), 0);
-
-    let removed_parallel_unit_id = workers
-        .iter()
-        .flat_map(|worker| {
-            worker
-                .parallel_units
-                .iter()
-                .map(|parallel_unit| parallel_unit.id)
-        })
-        .sorted()
-        .collect_vec();
-
-    assert_eq!(target_plan.removed_parallel_units, removed_parallel_unit_id);
-
-    Ok(())
-}
-#[tokio::test]
-async fn test_resize_single() -> Result<()> {
-    let mut cluster = Cluster::start(Configuration::for_scale()).await?;
-    let mut session = cluster.start_session();
-
-    session.run("create table t (v int);").await?;
-    session
-        .run("create materialized view mv1 as select count(*) from t;")
-        .await?;
-
-    session
-        .run("create materialized view mv2 as select * from mv1;")
-        .await?;
-
-    let agg_fragment = cluster
-        .locate_one_fragment([
-            identity_contains("simpleAgg"),
-            identity_contains("materialize"),
-        ])
-        .await?;
-
-    let agg_fragment_id = agg_fragment.inner.fragment_id;
-
-    let (_, used_parallel_unit_ids) = agg_fragment.parallel_unit_usage();
-
-    assert_eq!(used_parallel_unit_ids.len(), 1);
-
-    let used_parallel_unit_id = used_parallel_unit_ids.iter().next().unwrap();
-
-    let mut workers: Vec<WorkerNode> = cluster
-        .get_cluster_info()
-        .await?
-        .worker_nodes
-        .into_iter()
-        .filter(|worker| worker.r#type() == WorkerType::ComputeNode)
-        .collect();
-
-    let prev_workers = workers
-        .extract_if(|worker| {
-            worker
-                .parallel_units
-                .iter()
-                .map(|parallel_unit| parallel_unit.id)
-                .contains(used_parallel_unit_id)
-        })
-        .collect_vec();
-
-    let prev_worker = prev_workers.into_iter().exactly_one().unwrap();
-
-    let resp = cluster
-        .get_reschedule_plan(StableResizePolicy(PbStableResizePolicy {
-            fragment_worker_changes: HashMap::from([(
-                agg_fragment_id,
-                WorkerChanges {
-                    include_worker_ids: vec![],
-                    exclude_worker_ids: vec![prev_worker.id],
-                    ..Default::default()
-                },
-            )]),
-        }))
-        .await?;
-
-    let reschedules = resp.reschedules;
-    assert_eq!(reschedules.len(), 1);
-    let target_plan: PbReschedule = reschedules.get(&agg_fragment_id).unwrap().clone();
-    assert_eq!(target_plan.added_parallel_units.len(), 1);
-    assert_eq!(target_plan.removed_parallel_units.len(), 1);
-
-    let removed_parallel_unit_id = target_plan
-        .removed_parallel_units
-        .iter()
-        .exactly_one()
-        .unwrap();
-
-    assert!(prev_worker
-        .parallel_units
-        .iter()
-        .map(|parallel_unit| parallel_unit.id)
-        .contains(removed_parallel_unit_id));
-
-    Ok(())
-}
-
-#[tokio::test]
-async fn test_resize_single_failed() -> Result<()> {
-    let mut cluster = Cluster::start(Configuration::for_scale_no_shuffle()).await?;
-    let mut session = cluster.start_session();
-
-    session.run("create table t (v int);").await?;
-    session
-        .run("create materialized view mv1 as select count(*) from t;")
-        .await?;
-
-    session
-        .run("create materialized view mv2 as select * from mv1;")
-        .await?;
-
-    let upstream_fragment = cluster
-        .locate_one_fragment([
-            identity_contains("simpleAgg"),
-            identity_contains("materialize"),
-        ])
-        .await?;
-
-    let upstream_fragment_id = upstream_fragment.inner.fragment_id;
-
-    let downstream_fragment = cluster
-        .locate_one_fragment([
-            identity_contains("StreamTableScan"),
-            identity_contains("materialize"),
-        ])
-        .await?;
-
-    let downstream_fragment_id = downstream_fragment.inner.fragment_id;
-
-    let mut workers: Vec<WorkerNode> = cluster
-        .get_cluster_info()
-        .await?
-        .worker_nodes
-        .into_iter()
-        .filter(|worker| worker.r#type() == WorkerType::ComputeNode)
-        .collect();
-
-    let worker_a = workers.pop().unwrap();
-    let worker_b = workers.pop().unwrap();
-
-    let resp = cluster
-        .get_reschedule_plan(StableResizePolicy(PbStableResizePolicy {
-            fragment_worker_changes: HashMap::from([
-                (
-                    upstream_fragment_id,
-                    WorkerChanges {
-                        include_worker_ids: vec![],
-                        exclude_worker_ids: vec![worker_a.id],
-                        ..Default::default()
-                    },
-                ),
-                (
-                    downstream_fragment_id,
-                    WorkerChanges {
-                        include_worker_ids: vec![],
-                        exclude_worker_ids: vec![worker_b.id],
-                        ..Default::default()
-                    },
-                ),
-            ]),
-        }))
-        .await;
-
-    assert!(resp.is_err());
-
-    Ok(())
-}
-#[tokio::test]
-async fn test_resize_no_shuffle() -> Result<()> {
-    let mut cluster = Cluster::start(Configuration::for_scale_no_shuffle()).await?;
-    let mut session = cluster.start_session();
-
-    session.run("create table t (v int);").await?;
-    session
-        .run("create materialized view mv1 as select * from t;")
-        .await?;
-    session
-        .run("create materialized view mv2 as select * from t;")
-        .await?;
-    session
-        .run("create materialized view mv3 as select * from mv2;")
-        .await?;
-    session
-        .run("create materialized view mv4 as select * from mv2;")
-        .await?;
-    session
-        .run("create materialized view mv5 as select * from mv3;")
-        .await?;
-    session
-        .run("create materialized view mv6 as select * from mv3;")
-        .await?;
-    session
-        .run(
-            "create materialized view mv7 as select mv1.v as mv1v, mv5.v as mv5v from mv1
-join mv5 on mv1.v = mv5.v limit 1;",
-        )
-        .await?;
-
-    let chain_fragments: [_; 8] = cluster
-        .locate_fragments([identity_contains("StreamTableScan")])
-        .await?
-        .try_into()
-        .unwrap();
-
-    let selected_fragment = chain_fragments.choose(&mut rand::thread_rng()).unwrap();
-
-    let selected_fragment_id = selected_fragment.inner.fragment_id;
-
-    let mut workers: Vec<WorkerNode> = cluster
-        .get_cluster_info()
-        .await?
-        .worker_nodes
-        .into_iter()
-        .filter(|worker: &WorkerNode| worker.r#type() == WorkerType::ComputeNode)
-        .collect();
-
-    workers.pop();
-
-    let removed_worker_ids = workers.iter().map(|worker| worker.id).collect_vec();
-
-    let resp = cluster
-        .get_reschedule_plan(PbPolicy::StableResizePolicy(PbStableResizePolicy {
-            fragment_worker_changes: HashMap::from([(
-                selected_fragment_id,
-                WorkerChanges {
-                    include_worker_ids: vec![],
-                    exclude_worker_ids: removed_worker_ids,
-                    ..Default::default()
-                },
-            )]),
-        }))
-        .await?;
-
-    let reschedules = resp.reschedules;
-
-    assert_eq!(reschedules.len(), 1);
-
-    let top_materialize_fragment = cluster
-        .locate_one_fragment([
-            identity_contains("materialize"),
-            no_identity_contains("topn"),
-            no_identity_contains("StreamTableScan"),
-            no_identity_contains("hashJoin"),
-        ])
-        .await?;
-
-    let top_materialize_fragment_id = reschedules.keys().exactly_one().cloned().unwrap();
-
-    assert_eq!(
-        top_materialize_fragment_id,
-        top_materialize_fragment.inner.fragment_id
-    );
-
-    Ok(())
-}
diff --git a/src/tests/simulation/tests/integration_tests/scale/schedulability.rs b/src/tests/simulation/tests/integration_tests/scale/schedulability.rs
index edd0d02864b24..a8a6f73eedd3d 100644
--- a/src/tests/simulation/tests/integration_tests/scale/schedulability.rs
+++ b/src/tests/simulation/tests/integration_tests/scale/schedulability.rs
@@ -15,7 +15,7 @@
 use std::collections::HashSet;
 
 use anyhow::Result;
-use risingwave_common::hash::ParallelUnitId;
+use risingwave_common::hash::WorkerSlotId;
 use risingwave_pb::common::{WorkerNode, WorkerType};
 use risingwave_simulation::cluster::{Cluster, Configuration};
 
@@ -36,14 +36,10 @@ async fn test_cordon_normal() -> Result<()> {
         .collect();
 
     let cordoned_worker = workers.pop().unwrap();
-
-    let rest_parallel_unit_ids: HashSet<_> = workers
+    let rest_worker_slots: HashSet<_> = workers
         .iter()
         .flat_map(|worker| {
-            worker
-                .parallel_units
-                .iter()
-                .map(|parallel_unit| parallel_unit.id as ParallelUnitId)
+            (0..worker.parallelism).map(|idx| WorkerSlotId::new(worker.id, idx as _))
         })
         .collect();
 
@@ -54,9 +50,9 @@ async fn test_cordon_normal() -> Result<()> {
     let fragments = cluster.locate_fragments([]).await?;
 
     for fragment in fragments {
-        let (_, used) = fragment.parallel_unit_usage();
+        let used_worker_slots = fragment.used_worker_slots();
 
-        assert_eq!(used, rest_parallel_unit_ids);
+        assert_eq!(used_worker_slots, rest_worker_slots);
     }
 
     session.run("drop table t;").await?;
@@ -68,11 +64,9 @@ async fn test_cordon_normal() -> Result<()> {
     let fragments = cluster.locate_fragments([]).await?;
 
     for fragment in fragments {
-        let (all, used) = fragment.parallel_unit_usage();
-
-        let all: HashSet<_> = all.into_iter().collect();
-
-        assert_eq!(used, all);
+        let all_worker_slots = fragment.all_worker_slots();
+        let used_worker_slots = fragment.used_worker_slots();
+        assert_eq!(used_worker_slots, all_worker_slots);
     }
 
     Ok(())
diff --git a/src/tests/simulation/tests/integration_tests/scale/singleton_migration.rs b/src/tests/simulation/tests/integration_tests/scale/singleton_migration.rs
index 05e05f157913c..8b37cb7731baf 100644
--- a/src/tests/simulation/tests/integration_tests/scale/singleton_migration.rs
+++ b/src/tests/simulation/tests/integration_tests/scale/singleton_migration.rs
@@ -43,27 +43,24 @@ async fn test_singleton_migration() -> Result<()> {
         ])
         .await?;
 
-    let id = fragment.id();
+    let mut all_worker_slots = fragment.all_worker_slots().into_iter().collect_vec();
+    let used_worker_slots = fragment.used_worker_slots();
 
-    let (mut all, used) = fragment.parallel_unit_usage();
+    assert_eq!(used_worker_slots.len(), 1);
 
-    assert_eq!(used.len(), 1);
+    all_worker_slots.shuffle(&mut thread_rng());
 
-    all.shuffle(&mut thread_rng());
-
-    let mut target_parallel_units = all
+    let mut target_worker_slots = all_worker_slots
         .into_iter()
-        .filter(|parallel_unit_id| !used.contains(parallel_unit_id));
+        .filter(|work_slot| !used_worker_slots.contains(work_slot));
 
-    let source_parallel_unit = used.iter().next().cloned().unwrap();
-    let target_parallel_unit = target_parallel_units.next().unwrap();
+    let source_slot = used_worker_slots.iter().exactly_one().cloned().unwrap();
+    let target_slot = target_worker_slots.next().unwrap();
 
-    assert_ne!(target_parallel_unit, source_parallel_unit);
+    assert_ne!(target_slot, source_slot);
 
     cluster
-        .reschedule(format!(
-            "{id}-[{source_parallel_unit}]+[{target_parallel_unit}]"
-        ))
+        .reschedule(fragment.reschedule([source_slot], [target_slot]))
         .await?;
 
     sleep(Duration::from_secs(3)).await;
@@ -82,13 +79,11 @@ async fn test_singleton_migration() -> Result<()> {
         .await?
         .assert_result_eq("10");
 
-    let source_parallel_unit = target_parallel_unit;
-    let target_parallel_unit = target_parallel_units.next().unwrap();
+    let source_slot = target_slot;
+    let target_slot = target_worker_slots.next().unwrap();
 
     cluster
-        .reschedule(format!(
-            "{id}-[{source_parallel_unit}]+[{target_parallel_unit}]"
-        ))
+        .reschedule(fragment.reschedule([source_slot], [target_slot]))
         .await?;
 
     sleep(Duration::from_secs(3)).await;
diff --git a/src/tests/simulation/tests/integration_tests/scale/sink.rs b/src/tests/simulation/tests/integration_tests/scale/sink.rs
index a7cfa2ea2de6e..16621064c2464 100644
--- a/src/tests/simulation/tests/integration_tests/scale/sink.rs
+++ b/src/tests/simulation/tests/integration_tests/scale/sink.rs
@@ -17,6 +17,7 @@ use std::time::Duration;
 
 use anyhow::Result;
 use futures::StreamExt;
+use itertools::Itertools;
 use rand::prelude::SliceRandom;
 use rand::thread_rng;
 use rdkafka::consumer::{Consumer, StreamConsumer};
@@ -35,6 +36,7 @@ const DEBEZIUM_SINK_CREATE: &str = "create sink s2 from m with (connector='kafka
 const APPEND_ONLY_TOPIC: &str = "t_sink_append_only";
 const DEBEZIUM_TOPIC: &str = "t_sink_debezium";
 
+use risingwave_common::hash::WorkerSlotId;
 use serde_derive::{Deserialize, Serialize};
 
 #[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)]
@@ -114,11 +116,39 @@ async fn test_sink_append_only() -> Result<()> {
         ])
         .await?;
 
-    let id = materialize_fragment.id();
+    let workers = materialize_fragment
+        .all_worker_count()
+        .into_keys()
+        .collect_vec();
+
     check_kafka_after_insert(&mut cluster, &mut stream, &[1, 2, 3]).await?;
-    cluster.reschedule(format!("{id}-[1,2,3,4,5]")).await?;
+    cluster
+        .reschedule(materialize_fragment.reschedule(
+            [
+                WorkerSlotId::new(workers[0], 1),
+                WorkerSlotId::new(workers[1], 0),
+                WorkerSlotId::new(workers[1], 1),
+                WorkerSlotId::new(workers[2], 0),
+                WorkerSlotId::new(workers[2], 1),
+            ],
+            [],
+        ))
+        .await?;
+
     check_kafka_after_insert(&mut cluster, &mut stream, &[4, 5, 6]).await?;
-    cluster.reschedule(format!("{id}+[1,2,3,4,5]")).await?;
+    cluster
+        .reschedule(materialize_fragment.reschedule(
+            [],
+            [
+                WorkerSlotId::new(workers[0], 1),
+                WorkerSlotId::new(workers[1], 0),
+                WorkerSlotId::new(workers[1], 1),
+                WorkerSlotId::new(workers[2], 0),
+                WorkerSlotId::new(workers[2], 1),
+            ],
+        ))
+        .await?;
+
     check_kafka_after_insert(&mut cluster, &mut stream, &[7, 8, 9]).await?;
 
     Ok(())
@@ -168,36 +198,34 @@ async fn test_sink_debezium() -> Result<()> {
         ])
         .await?;
 
-    let (mut all, used) = materialize_fragment.parallel_unit_usage();
+    let mut all_worker_slots = materialize_fragment
+        .all_worker_slots()
+        .into_iter()
+        .collect_vec();
+    let used_worker_slots = materialize_fragment.used_worker_slots();
 
-    assert_eq!(used.len(), 1);
+    assert_eq!(used_worker_slots.len(), 1);
 
-    all.shuffle(&mut thread_rng());
+    all_worker_slots.shuffle(&mut thread_rng());
 
-    let mut target_parallel_units = all
+    let mut target_worker_slots = all_worker_slots
         .into_iter()
-        .filter(|parallel_unit_id| !used.contains(parallel_unit_id));
-
-    let id = materialize_fragment.id();
+        .filter(|worker_slot| !used_worker_slots.contains(worker_slot));
 
     check_kafka_after_insert(&mut cluster, &mut stream, &[1, 2, 3]).await?;
 
-    let source_parallel_unit = used.iter().next().cloned().unwrap();
-    let target_parallel_unit = target_parallel_units.next().unwrap();
+    let source_slot = used_worker_slots.iter().next().cloned().unwrap();
+    let target_slot = target_worker_slots.next().unwrap();
     cluster
-        .reschedule(format!(
-            "{id}-[{source_parallel_unit}]+[{target_parallel_unit}]"
-        ))
+        .reschedule(materialize_fragment.reschedule([source_slot], [target_slot]))
         .await?;
     check_kafka_after_insert(&mut cluster, &mut stream, &[4, 5, 6]).await?;
 
-    let source_parallel_unit = target_parallel_unit;
-    let target_parallel_unit = target_parallel_units.next().unwrap();
+    let source_slot = target_slot;
+    let target_slot = target_worker_slots.next().unwrap();
 
     cluster
-        .reschedule(format!(
-            "{id}-[{source_parallel_unit}]+[{target_parallel_unit}]"
-        ))
+        .reschedule(materialize_fragment.reschedule([source_slot], [target_slot]))
         .await?;
     check_kafka_after_insert(&mut cluster, &mut stream, &[7, 8, 9]).await?;
 
diff --git a/src/tests/simulation/tests/integration_tests/scale/table.rs b/src/tests/simulation/tests/integration_tests/scale/table.rs
index 6bceb5c07b536..85a3e64d64108 100644
--- a/src/tests/simulation/tests/integration_tests/scale/table.rs
+++ b/src/tests/simulation/tests/integration_tests/scale/table.rs
@@ -16,6 +16,7 @@ use std::iter::repeat_with;
 
 use anyhow::Result;
 use itertools::Itertools;
+use risingwave_common::hash::WorkerSlotId;
 use risingwave_simulation::cluster::{Cluster, Configuration};
 use risingwave_simulation::ctl_ext::predicate::identity_contains;
 
@@ -47,13 +48,30 @@ async fn test_table() -> Result<()> {
 
     insert_and_flush!(cluster);
 
+    let workers = fragment.all_worker_count().into_keys().collect_vec();
+
     cluster
-        .reschedule(fragment.reschedule([0, 2, 4], []))
+        .reschedule(fragment.reschedule(
+            [
+                WorkerSlotId::new(workers[0], 0),
+                WorkerSlotId::new(workers[1], 0),
+                WorkerSlotId::new(workers[2], 0),
+            ],
+            [],
+        ))
         .await?;
 
     insert_and_flush!(cluster);
 
-    cluster.reschedule(fragment.reschedule([1], [0, 4])).await?;
+    cluster
+        .reschedule(fragment.reschedule(
+            [WorkerSlotId::new(workers[0], 1)],
+            [
+                WorkerSlotId::new(workers[0], 0),
+                WorkerSlotId::new(workers[2], 0),
+            ],
+        ))
+        .await?;
 
     insert_and_flush!(cluster);
 
@@ -69,13 +87,30 @@ async fn test_mv_on_scaled_table() -> Result<()> {
         .locate_one_fragment([identity_contains("materialize")])
         .await?;
 
+    let workers = fragment.all_worker_count().into_keys().collect_vec();
+
     cluster
-        .reschedule(fragment.reschedule([0, 2, 4], []))
+        .reschedule(fragment.reschedule(
+            [
+                WorkerSlotId::new(workers[0], 0),
+                WorkerSlotId::new(workers[1], 0),
+                WorkerSlotId::new(workers[2], 0),
+            ],
+            [],
+        ))
         .await?;
 
     insert_and_flush!(cluster);
 
-    cluster.reschedule(fragment.reschedule([1], [0, 4])).await?;
+    cluster
+        .reschedule(fragment.reschedule(
+            [WorkerSlotId::new(workers[0], 1)],
+            [
+                WorkerSlotId::new(workers[0], 0),
+                WorkerSlotId::new(workers[2], 0),
+            ],
+        ))
+        .await?;
 
     insert_and_flush!(cluster);
 
@@ -97,8 +132,17 @@ async fn test_scale_on_schema_change() -> Result<()> {
         .locate_one_fragment([identity_contains("materialize"), identity_contains("union")])
         .await?;
 
+    let workers = fragment.all_worker_count().into_keys().collect_vec();
+
     cluster
-        .reschedule(fragment.reschedule([0, 2, 4], []))
+        .reschedule(fragment.reschedule(
+            [
+                WorkerSlotId::new(workers[0], 0),
+                WorkerSlotId::new(workers[1], 0),
+                WorkerSlotId::new(workers[2], 0),
+            ],
+            [],
+        ))
         .await?;
 
     insert_and_flush!(cluster);
@@ -113,13 +157,19 @@ async fn test_scale_on_schema_change() -> Result<()> {
         .await?;
 
     cluster
-        .reschedule_resolve_no_shuffle(fragment.reschedule([1], [0, 4]))
+        .reschedule_resolve_no_shuffle(fragment.reschedule(
+            [WorkerSlotId::new(workers[0], 1)],
+            [
+                WorkerSlotId::new(workers[0], 0),
+                WorkerSlotId::new(workers[2], 0),
+            ],
+        ))
         .await?;
 
     let fragment = cluster
         .locate_one_fragment([identity_contains("materialize"), identity_contains("union")])
         .await?;
-    let (_, used) = fragment.parallel_unit_usage();
+    let used = fragment.used_worker_slots();
     assert_eq!(used.len(), 4);
 
     insert_and_flush!(cluster);
diff --git a/src/tests/simulation/tests/integration_tests/sink/recovery.rs b/src/tests/simulation/tests/integration_tests/sink/recovery.rs
index 2de83b424ad42..6b4f71d7d508e 100644
--- a/src/tests/simulation/tests/integration_tests/sink/recovery.rs
+++ b/src/tests/simulation/tests/integration_tests/sink/recovery.rs
@@ -40,7 +40,7 @@ async fn kill_and_check(
             break;
         }
         assert!(
-            curr_count > prev_count,
+            curr_count >= prev_count,
             "not make progress between kill. Prev count {}, curr count {}, i {}",
             prev_count,
             curr_count,
diff --git a/src/tests/simulation/tests/integration_tests/sink/scale.rs b/src/tests/simulation/tests/integration_tests/sink/scale.rs
index 78cc6905eed67..b1c533b21cab3 100644
--- a/src/tests/simulation/tests/integration_tests/sink/scale.rs
+++ b/src/tests/simulation/tests/integration_tests/sink/scale.rs
@@ -19,6 +19,7 @@ use std::time::Duration;
 use anyhow::Result;
 use itertools::Itertools;
 use rand::{thread_rng, Rng};
+use risingwave_common::hash::WorkerSlotId;
 use risingwave_simulation::cluster::{Cluster, KillOpts};
 use risingwave_simulation::ctl_ext::predicate::identity_contains;
 use tokio::time::sleep;
@@ -79,19 +80,55 @@ async fn scale_test_inner(is_decouple: bool) -> Result<()> {
         .await?;
 
     assert_eq!(sink_fragments.len(), 1);
-    let framgment = sink_fragments.pop().unwrap();
-    let id = framgment.id();
+    let fragment = sink_fragments.pop().unwrap();
+    let id = fragment.id();
 
     let count = test_source.id_list.len();
+    let workers = fragment.all_worker_count().into_keys().collect_vec();
 
     scale_and_check(
         &mut cluster,
         &test_sink,
         count,
         vec![
-            (format!("{id}-[1,2,3]"), 3),
-            (format!("{id}-[4,5]+[1,2]"), 3),
-            (format!("{id}+[3,4,5]"), 6),
+            // (format!("{id}-[1,2,3]"), 3),
+            (
+                fragment.reschedule(
+                    [
+                        WorkerSlotId::new(workers[0], 0),
+                        WorkerSlotId::new(workers[1], 0),
+                        WorkerSlotId::new(workers[1], 1),
+                    ],
+                    [],
+                ),
+                3,
+            ),
+            // (format!("{id}-[4,5]+[1,2]"), 3)
+            (
+                fragment.reschedule(
+                    [
+                        WorkerSlotId::new(workers[2], 0),
+                        WorkerSlotId::new(workers[2], 1),
+                    ],
+                    [
+                        WorkerSlotId::new(workers[0], 1),
+                        WorkerSlotId::new(workers[1], 0),
+                    ],
+                ),
+                3,
+            ),
+            // (format!("{id}+[3,4,5]"), 6),
+            (
+                fragment.reschedule(
+                    [],
+                    [
+                        WorkerSlotId::new(workers[1], 1),
+                        WorkerSlotId::new(workers[2], 0),
+                        WorkerSlotId::new(workers[2], 1),
+                    ],
+                ),
+                6,
+            ),
         ]
         .into_iter(),
     )
diff --git a/src/tests/sqlsmith/Cargo.toml b/src/tests/sqlsmith/Cargo.toml
index f0b4b425a2af5..8f4fe66abf052 100644
--- a/src/tests/sqlsmith/Cargo.toml
+++ b/src/tests/sqlsmith/Cargo.toml
@@ -27,7 +27,7 @@ risingwave_expr_impl = { workspace = true }
 risingwave_frontend = { workspace = true }
 risingwave_pb = { workspace = true }
 risingwave_sqlparser = { workspace = true }
-similar = "2.5.0"
+similar = "2.6.0"
 thiserror-ext = { workspace = true }
 tokio = { version = "0.2", package = "madsim-tokio" }
 tokio-postgres = "0.7"
diff --git a/src/tests/sqlsmith/src/lib.rs b/src/tests/sqlsmith/src/lib.rs
index 0c215c9146a45..2eb9c59d1bf38 100644
--- a/src/tests/sqlsmith/src/lib.rs
+++ b/src/tests/sqlsmith/src/lib.rs
@@ -14,7 +14,6 @@
 
 #![feature(let_chains)]
 #![feature(if_let_guard)]
-#![feature(lazy_cell)]
 #![feature(box_patterns)]
 #![feature(register_tool)]
 #![register_tool(rw)]
diff --git a/src/tests/sqlsmith/src/sql_gen/agg.rs b/src/tests/sqlsmith/src/sql_gen/agg.rs
index 441e71e86c9f0..4953235d4cba4 100644
--- a/src/tests/sqlsmith/src/sql_gen/agg.rs
+++ b/src/tests/sqlsmith/src/sql_gen/agg.rs
@@ -15,7 +15,7 @@
 use rand::seq::SliceRandom;
 use rand::Rng;
 use risingwave_common::types::DataType;
-use risingwave_expr::aggregate::AggKind;
+use risingwave_expr::aggregate::PbAggKind;
 use risingwave_expr::sig::SigDataType;
 use risingwave_sqlparser::ast::{
     Expr, Function, FunctionArg, FunctionArgExpr, Ident, ObjectName, OrderByExpr,
@@ -31,7 +31,7 @@ impl<'a, R: Rng> SqlGenerator<'a, R> {
             Some(funcs) => funcs,
         };
         let func = funcs.choose(&mut self.rng).unwrap();
-        if matches!(func.name.as_aggregate(), AggKind::Min | AggKind::Max)
+        if matches!(func.name.as_aggregate(), PbAggKind::Min | PbAggKind::Max)
             && matches!(
                 func.ret_type,
                 SigDataType::Exact(DataType::Boolean | DataType::Jsonb)
@@ -51,7 +51,7 @@ impl<'a, R: Rng> SqlGenerator<'a, R> {
         // DISTINCT now only works with agg kinds except `ApproxCountDistinct`, and with at least
         // one argument and only the first being non-constant. See `Binder::bind_normal_agg`
         // for more details.
-        let distinct_allowed = func.name.as_aggregate() != AggKind::ApproxCountDistinct
+        let distinct_allowed = func.name.as_aggregate() != PbAggKind::ApproxCountDistinct
             && !exprs.is_empty()
             && exprs.iter().skip(1).all(|e| matches!(e, Expr::Value(_)));
         let distinct = distinct_allowed && self.flip_coin();
@@ -86,13 +86,13 @@ impl<'a, R: Rng> SqlGenerator<'a, R> {
     /// Generates aggregate expressions. For internal / unsupported aggregators, we return `None`.
     fn make_agg_expr(
         &mut self,
-        func: AggKind,
+        func: PbAggKind,
         exprs: &[Expr],
         distinct: bool,
         filter: Option<Box<Expr>>,
         order_by: Vec<OrderByExpr>,
     ) -> Option<Expr> {
-        use AggKind as A;
+        use PbAggKind as A;
         match func {
             kind @ (A::FirstValue | A::LastValue) => {
                 if order_by.is_empty() {
@@ -100,7 +100,7 @@ impl<'a, R: Rng> SqlGenerator<'a, R> {
                     None
                 } else {
                     Some(Expr::Function(make_agg_func(
-                        &kind.to_string(),
+                        &kind.as_str_name().to_lowercase(),
                         exprs,
                         distinct,
                         filter,
@@ -109,7 +109,7 @@ impl<'a, R: Rng> SqlGenerator<'a, R> {
                 }
             }
             other => Some(Expr::Function(make_agg_func(
-                &other.to_string(),
+                &other.as_str_name().to_lowercase(),
                 exprs,
                 distinct,
                 filter,
@@ -140,6 +140,7 @@ fn make_agg_func(
     };
 
     Function {
+        scalar_as_agg: false,
         name: ObjectName(vec![Ident::new_unchecked(func_name)]),
         args,
         variadic: false,
diff --git a/src/tests/sqlsmith/src/sql_gen/functions.rs b/src/tests/sqlsmith/src/sql_gen/functions.rs
index 332054ad3f66c..cee18a18081ca 100644
--- a/src/tests/sqlsmith/src/sql_gen/functions.rs
+++ b/src/tests/sqlsmith/src/sql_gen/functions.rs
@@ -256,6 +256,7 @@ pub fn make_simple_func(func_name: &str, exprs: &[Expr]) -> Function {
         .collect();
 
     Function {
+        scalar_as_agg: false,
         name: ObjectName(vec![Ident::new_unchecked(func_name)]),
         args,
         variadic: false,
diff --git a/src/tests/sqlsmith/src/sql_gen/types.rs b/src/tests/sqlsmith/src/sql_gen/types.rs
index 141accc71abc2..2f12f96790f3d 100644
--- a/src/tests/sqlsmith/src/sql_gen/types.rs
+++ b/src/tests/sqlsmith/src/sql_gen/types.rs
@@ -19,7 +19,7 @@ use std::sync::LazyLock;
 
 use itertools::Itertools;
 use risingwave_common::types::{DataType, DataTypeName};
-use risingwave_expr::aggregate::AggKind;
+use risingwave_expr::aggregate::PbAggKind;
 use risingwave_expr::sig::{FuncSign, FUNCTION_REGISTRY};
 use risingwave_frontend::expr::{cast_sigs, CastContext, CastSig as RwCastSig, ExprType};
 use risingwave_sqlparser::ast::{BinaryOperator, DataType as AstDataType, StructField};
@@ -176,18 +176,19 @@ pub(crate) static AGG_FUNC_TABLE: LazyLock<HashMap<DataType, Vec<&'static FuncSi
                     && func.ret_type.is_exact()
                     // Ignored functions
                     && ![
-                        AggKind::InternalLastSeenValue, // Use internally
-                        AggKind::Sum0, // Used internally
-                        AggKind::BitAnd,
-                        AggKind::BitOr,
-                        AggKind::BoolAnd,
-                        AggKind::BoolOr,
-                        AggKind::PercentileCont,
-                        AggKind::PercentileDisc,
-                        AggKind::Mode,
-                        AggKind::JsonbObjectAgg, // ENABLE: https://github.com/risingwavelabs/risingwave/issues/16293
-                        AggKind::StddevSamp, // ENABLE: https://github.com/risingwavelabs/risingwave/issues/16293
-                        AggKind::VarSamp, // ENABLE: https://github.com/risingwavelabs/risingwave/issues/16293
+                        PbAggKind::InternalLastSeenValue, // Use internally
+                        PbAggKind::Sum0, // Used internally
+                        PbAggKind::BitAnd,
+                        PbAggKind::BitOr,
+                        PbAggKind::BoolAnd,
+                        PbAggKind::BoolOr,
+                        PbAggKind::PercentileCont,
+                        PbAggKind::PercentileDisc,
+                        PbAggKind::Mode,
+                        PbAggKind::ApproxPercentile, // ENABLE: https://github.com/risingwavelabs/risingwave/issues/16293
+                        PbAggKind::JsonbObjectAgg, // ENABLE: https://github.com/risingwavelabs/risingwave/issues/16293
+                        PbAggKind::StddevSamp, // ENABLE: https://github.com/risingwavelabs/risingwave/issues/16293
+                        PbAggKind::VarSamp, // ENABLE: https://github.com/risingwavelabs/risingwave/issues/16293
                     ]
                     .contains(&func.name.as_aggregate())
                     // Exclude 2 phase agg global sum.
@@ -195,7 +196,7 @@ pub(crate) static AGG_FUNC_TABLE: LazyLock<HashMap<DataType, Vec<&'static FuncSi
                     // Otherwise it conflicts with normal aggregation:
                     // Sum(Int64) -> Decimal.
                     // And sqlsmith will generate expressions with wrong types.
-                    && if func.name.as_aggregate() == AggKind::Sum {
+                    && if func.name.as_aggregate() == PbAggKind::Sum {
                        !(func.inputs_type[0].as_exact() == &DataType::Int64 && func.ret_type.as_exact() == &DataType::Int64)
                     } else {
                        true
diff --git a/src/utils/pgwire/Cargo.toml b/src/utils/pgwire/Cargo.toml
index 1744fab18d250..0d29afea01855 100644
--- a/src/utils/pgwire/Cargo.toml
+++ b/src/utils/pgwire/Cargo.toml
@@ -22,7 +22,7 @@ bytes = "1"
 futures = { version = "0.3", default-features = false, features = ["alloc"] }
 itertools = { workspace = true }
 jsonwebtoken = "9"
-openssl = "0.10.60"
+openssl = "0.10.66"
 panic-message = "0.3"
 parking_lot = { workspace = true }
 reqwest = "0.12.2"
diff --git a/src/utils/pgwire/src/lib.rs b/src/utils/pgwire/src/lib.rs
index 2279156d89b37..8d1c00541bb95 100644
--- a/src/utils/pgwire/src/lib.rs
+++ b/src/utils/pgwire/src/lib.rs
@@ -16,7 +16,6 @@
 #![feature(trait_alias)]
 #![feature(iterator_try_collect)]
 #![feature(trusted_len)]
-#![feature(lazy_cell)]
 #![feature(buf_read_has_data_left)]
 #![feature(round_char_boundary)]
 #![feature(never_type)]
diff --git a/src/utils/pgwire/src/types.rs b/src/utils/pgwire/src/types.rs
index d4d37e1168ea1..c76aa20aac4cd 100644
--- a/src/utils/pgwire/src/types.rs
+++ b/src/utils/pgwire/src/types.rs
@@ -59,6 +59,7 @@ impl Index<usize> for Row {
     }
 }
 
+/// <https://www.postgresql.org/docs/current/protocol-overview.html#PROTOCOL-FORMAT-CODES>
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub enum Format {
     Binary,
diff --git a/src/utils/runtime/src/lib.rs b/src/utils/runtime/src/lib.rs
index 6418c18ea103a..75a4a5f4d00b1 100644
--- a/src/utils/runtime/src/lib.rs
+++ b/src/utils/runtime/src/lib.rs
@@ -35,6 +35,7 @@ mod panic_hook;
 pub use panic_hook::*;
 mod prof;
 use prof::*;
+use tokio::signal::unix::SignalKind;
 
 /// Start RisingWave components with configs from environment variable.
 ///
@@ -100,20 +101,22 @@ where
         let shutdown = CancellationToken::new();
         let mut fut = pin!(f(shutdown.clone()));
 
+        let mut sigint = tokio::signal::unix::signal(SignalKind::interrupt()).unwrap();
+        let mut sigterm = tokio::signal::unix::signal(SignalKind::terminate()).unwrap();
+
         tokio::select! {
             biased;
-            result = tokio::signal::ctrl_c() => {
-                result.expect("failed to receive ctrl-c signal");
-                tracing::info!("received ctrl-c, shutting down... (press ctrl-c again to force shutdown)");
 
-                // Send shutdown signal.
+            // Watch SIGINT, typically originating from user pressing ctrl-c.
+            // Attempt to shutdown gracefully and force shutdown on the next signal.
+            _ = sigint.recv() => {
+                tracing::info!("received ctrl-c, shutting down... (press ctrl-c again to force shutdown)");
                 shutdown.cancel();
 
                 // While waiting for the future to finish, listen for the second ctrl-c signal.
                 tokio::select! {
                     biased;
-                    result = tokio::signal::ctrl_c() => {
-                        result.expect("failed to receive ctrl-c signal");
+                    _ = sigint.recv() => {
                         tracing::warn!("forced shutdown");
 
                         // Directly exit the process **here** instead of returning from the future, since
@@ -123,6 +126,16 @@ where
                     _ = &mut fut => {},
                 }
             }
+
+            // Watch SIGTERM, typically originating from Kubernetes.
+            // Attempt to shutdown gracefully. No need to force shutdown since it will send SIGKILL after a timeout.
+            _ = sigterm.recv() => {
+                tracing::info!("received SIGTERM, shutting down...");
+                shutdown.cancel();
+                fut.await;
+            }
+
+            // Proceed with the future.
             _ = &mut fut => {},
         }
     };